enigma-memory 0.1.17 → 0.1.22

package/packages/rag/src/embeddings.js

@@ -0,0 +1,459 @@
+// Enigma Sovereign Embedding & Lexical-Vector Provider.
+// Neural inference is local-only and pinned to the bundled, hash-verified model.
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+
+export const EMBEDDING_DIM = 384;
+
+export const LEXICAL_HASH_DESCRIPTOR = Object.freeze({
+  provider: 'enigma_lexical_hash',
+  version: '1.0.0',
+  dimension: EMBEDDING_DIM,
+  architecture: 'DeterministicLexicalHash/NgramProjection',
+  description: 'Explicit deterministic subword n-gram fallback with position-weighted pooling and L2 unit-sphere normalization.',
+});
+
+export const LOCAL_MINILM_DESCRIPTOR = Object.freeze({
+  provider: 'huggingface_transformers_local',
+  package: '@huggingface/transformers',
+  packageVersion: '4.2.0',
+  modelId: 'Xenova/all-MiniLM-L6-v2',
+  immutableCommit: '751bff37182d3f1213fa05d7196b954e230abad9',
+  dimension: EMBEDDING_DIM,
+  maxSequenceLength: 512,
+  dtype: 'q8',
+  device: 'cpu',
+  pooling: 'mean',
+  normalization: 'l2_unit_sphere',
+  manifest: 'sha256-manifest.json',
+  manifestSha256: 'c1d03ad7b6e33e74fdbb862bda766cffb6c2bc6674f9628172d98c8f58b8b103',
+  artifacts: Object.freeze([
+    Object.freeze({ path: 'config.json', size: 650, sha256: '7135149f7cffa1a573466c6e4d8423ed73b62fd2332c575bf738a0d033f70df7' }),
+    Object.freeze({ path: 'tokenizer.json', size: 711661, sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' }),
+    Object.freeze({ path: 'tokenizer_config.json', size: 366, sha256: '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3' }),
+    Object.freeze({ path: 'onnx/model_quantized.onnx', size: 22972370, sha256: 'afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1' }),
+  ]),
+});
+
+// Retained name for callers that inspected the earlier integration descriptor.
+export const ONNX_INTEGRATION_SPEC = LOCAL_MINILM_DESCRIPTOR;
+
+/**
+ * Word and subword tokenizer.
+ * Segments strings into lowercase words and subword fragments.
+ * @param {string} text
+ * @returns {Array<{ token: string, isSubword: boolean, pos: number }>}
+ */
+export function tokenizeSubwords(text) {
+  if (!text || typeof text !== 'string') return [];
+  const normalized = text.toLowerCase().replace(/[^\w\s]/g, ' ');
+  const words = normalized.split(/\s+/).filter(w => w.length > 0);
+  const tokens = [];
+
+  let pos = 0;
+  for (const word of words) {
+    tokens.push({ token: word, isSubword: false, pos: pos++ });
+    if (word.length > 4) {
+      const mid = Math.floor(word.length / 2);
+      tokens.push({ token: word.slice(0, mid), isSubword: false, pos });
+      tokens.push({ token: `##${word.slice(mid)}`, isSubword: true, pos });
+      pos++;
+    }
+  }
+  return tokens;
+}
+
+/**
+ * Deterministic Murmur-style hash for feature projection.
+ */
+function hashFeature(str, seed = 0) {
+  let h = seed ^ str.length;
+  for (let i = 0; i < str.length; i++) {
+    const c = str.charCodeAt(i);
+    h = Math.imul(h ^ c, 0x5bd1e995);
+    h ^= h >>> 15;
+  }
+  return (h >>> 0);
+}
+
+/**
+ * Computes 384-dimensional dense vector via deterministic subword feature hashing,
+ * position-weighted pooling, and L2 unit-sphere normalization.
+ * Note: This is an offline lexical-hash fallback, not neural transformer inference.
+ * @param {string} text
+ * @param {number} [dim=384]
+ * @returns {Float32Array} L2-normalized 384-D vector
+ */
+export function computeLexicalEmbedding(text, dim = EMBEDDING_DIM) {
+  const tokens = tokenizeSubwords(text);
+  const vector = new Float32Array(dim);
+
+  if (tokens.length === 0) {
+    return vector;
+  }
+
+  const HEAD_SEEDS = [0x9747b28c, 0xcafebabe, 0x85ebca6b, 0x5bd1e995, 0x12b9b0a1, 0x43f119e2];
+  let totalWeight = 0;
+
+  for (const item of tokens) {
+    const weight = (item.isSubword ? 0.65 : 1.25) / Math.sqrt(1 + item.pos * 0.05);
+    totalWeight += weight;
+
+    for (let h = 0; h < HEAD_SEEDS.length; h++) {
+      const h1 = hashFeature(item.token, HEAD_SEEDS[h]);
+      const idx = h1 % dim;
+      const sign = (hashFeature(item.token, h1) & 1) === 0 ? 1.0 : -1.0;
+      vector[idx] += sign * weight;
+    }
+  }
+
+  if (totalWeight > 1e-12) {
+    const invWeight = 1.0 / totalWeight;
+    for (let i = 0; i < dim; i++) {
+      vector[i] *= invWeight;
+    }
+  }
+
+  const mixed = new Float32Array(dim);
+  for (let i = 0; i < dim; i++) {
+    const prev = vector[(i - 1 + dim) % dim];
+    const curr = vector[i];
+    const next = vector[(i + 1) % dim];
+    mixed[i] = curr * 0.70 + (prev + next) * 0.15;
+  }
+
+  let normSq = 0;
+  for (let i = 0; i < dim; i++) {
+    normSq += mixed[i] * mixed[i];
+  }
+  const norm = Math.sqrt(normSq);
+  if (norm > 1e-12) {
+    const invNorm = 1.0 / norm;
+    for (let i = 0; i < dim; i++) {
+      mixed[i] *= invNorm;
+    }
+  }
+
+  return mixed;
+}
+
+const BUNDLED_MODELS_URL = new URL('../models/', import.meta.url);
+const BUNDLED_MODEL_URL = new URL(`${LOCAL_MINILM_DESCRIPTOR.modelId}/`, BUNDLED_MODELS_URL);
+
+let sharedExtractorPromise = null;
+let sharedExtractorLeases = 0;
+let sharedExtractorDisposal = null;
+
+export async function verifyBundledModel() {
+  const manifestUrl = new URL(LOCAL_MINILM_DESCRIPTOR.manifest, BUNDLED_MODEL_URL);
+  let manifestBytes;
+  try {
+    manifestBytes = await readFile(manifestUrl);
+  } catch (error) {
+    throw new Error('Bundled embedding model SHA-256 manifest is unavailable', { cause: error });
+  }
+
+  const manifestHash = createHash('sha256').update(manifestBytes).digest('hex');
+  if (manifestHash !== LOCAL_MINILM_DESCRIPTOR.manifestSha256) {
+    throw new Error(
+      `Bundled embedding model SHA-256 manifest hash mismatch: expected ${LOCAL_MINILM_DESCRIPTOR.manifestSha256}, received ${manifestHash}`
+    );
+  }
+
+  let manifest;
+  try {
+    manifest = JSON.parse(manifestBytes.toString('utf8'));
+  } catch (error) {
+    throw new Error('Bundled embedding model SHA-256 manifest is invalid JSON', { cause: error });
+  }
+  if (
+    manifest.schemaVersion !== 1
+    || manifest.algorithm !== 'sha256'
+    || manifest.modelId !== LOCAL_MINILM_DESCRIPTOR.modelId
+    || manifest.revision !== LOCAL_MINILM_DESCRIPTOR.immutableCommit
+    || !Array.isArray(manifest.artifacts)
+    || manifest.artifacts.length !== LOCAL_MINILM_DESCRIPTOR.artifacts.length
+  ) {
+    throw new Error('Bundled embedding model SHA-256 manifest does not match the pinned runtime descriptor');
+  }
+
+  for (let index = 0; index < LOCAL_MINILM_DESCRIPTOR.artifacts.length; index += 1) {
+    const artifact = LOCAL_MINILM_DESCRIPTOR.artifacts[index];
+    const manifestArtifact = manifest.artifacts[index];
+    if (
+      manifestArtifact?.path !== artifact.path
+      || manifestArtifact?.size !== artifact.size
+      || manifestArtifact?.sha256 !== artifact.sha256
+    ) {
+      throw new Error(`Bundled embedding model SHA-256 manifest entry mismatch for ${artifact.path}`);
+    }
+
+    const artifactUrl = new URL(artifact.path, BUNDLED_MODEL_URL);
+    let bytes;
+    try {
+      bytes = await readFile(artifactUrl);
+    } catch (error) {
+      throw new Error(`Bundled embedding model artifact is unavailable: ${artifact.path}`, { cause: error });
+    }
+
+    if (bytes.byteLength !== artifact.size) {
+      throw new Error(
+        `Bundled embedding model artifact size mismatch for ${artifact.path}: expected ${artifact.size}, received ${bytes.byteLength}`
+      );
+    }
+    const actualHash = createHash('sha256').update(bytes).digest('hex');
+    if (actualHash !== artifact.sha256) {
+      throw new Error(
+        `Bundled embedding model artifact hash mismatch for ${artifact.path}: expected ${artifact.sha256}, received ${actualHash}`
+      );
+    }
+  }
+}
+
+async function loadSharedExtractor() {
+  if (sharedExtractorDisposal) await sharedExtractorDisposal;
+  if (!sharedExtractorPromise) {
+    sharedExtractorPromise = (async () => {
+      await verifyBundledModel();
+      const { env, pipeline } = await import('@huggingface/transformers');
+      env.localModelPath = fileURLToPath(BUNDLED_MODELS_URL);
+      env.allowLocalModels = true;
+      env.allowRemoteModels = false;
+      env.useFS = true;
+      env.useBrowserCache = false;
+      env.useFSCache = false;
+      env.cacheDir = null;
+      env.useCustomCache = false;
+      env.customCache = null;
+      env.useWasmCache = false;
+      const extractor = await pipeline(
+        'feature-extraction',
+        LOCAL_MINILM_DESCRIPTOR.modelId,
+        {
+          revision: LOCAL_MINILM_DESCRIPTOR.immutableCommit,
+          local_files_only: true,
+          device: LOCAL_MINILM_DESCRIPTOR.device,
+          dtype: LOCAL_MINILM_DESCRIPTOR.dtype,
+        }
+      );
+      return extractor;
+    })().catch((error) => {
+      sharedExtractorPromise = null;
+      throw error;
+    });
+  }
+  return await sharedExtractorPromise;
+}
+
+async function releaseSharedExtractor() {
+  sharedExtractorLeases -= 1;
+  if (sharedExtractorLeases !== 0 || !sharedExtractorPromise) return;
+
+  const extractorPromise = sharedExtractorPromise;
+  sharedExtractorPromise = null;
+  sharedExtractorDisposal = (async () => {
+    const extractor = await extractorPromise;
+    await extractor.dispose();
+  })().finally(() => {
+    sharedExtractorDisposal = null;
+  });
+  await sharedExtractorDisposal;
+}
+
+function coerceAdapterVector(output, dimension) {
+  const candidate = Array.isArray(output) && output.length === 1
+    && (Array.isArray(output[0]) || ArrayBuffer.isView(output[0]))
+    ? output[0]
+    : output;
+  const vector = candidate instanceof Float32Array
+    ? new Float32Array(candidate)
+    : Array.isArray(candidate)
+      ? new Float32Array(candidate)
+      : null;
+  if (!vector) {
+    throw new TypeError('ONNX runner must return a Float32Array, number array, or one-element batch');
+  }
+  if (vector.length !== dimension) {
+    throw new Error(`ONNX runner returned ${vector.length} values; expected ${dimension}`);
+  }
+  return vector;
+}
+
+function createAdapterProvider(onnxRunner, dimension) {
+  const invoke = typeof onnxRunner === 'function'
+    ? (text) => onnxRunner(text)
+    : onnxRunner && typeof onnxRunner.run === 'function'
+      ? async (text) => {
+          const result = await onnxRunner.run([text]);
+          return result;
+        }
+      : null;
+  if (!invoke) throw new TypeError('onnxRunner must be a function or an object with run(texts)');
+
+  return {
+    providerType: 'onnx_runtime_adapter',
+    descriptor: LOCAL_MINILM_DESCRIPTOR,
+    dimension,
+    async embed(text) {
+      return coerceAdapterVector(await invoke(text), dimension);
+    },
+    async embedBatch(texts) {
+      return await Promise.all(texts.map((text) => this.embed(text)));
+    },
+    embedSync() {
+      throw new Error('embedSync() is unsupported by an async ONNX runner; use await embed() instead');
+    },
+    async dispose() {
+      await onnxRunner.dispose?.();
+    },
+  };
+}
+
+function createLocalNeuralProvider(dimension) {
+  if (dimension !== EMBEDDING_DIM) {
+    throw new Error(`The bundled MiniLM model requires dimension ${EMBEDDING_DIM}, received ${dimension}`);
+  }
+
+  let extractorLeasePromise = null;
+  let disposed = false;
+  const acquire = async () => {
+    if (disposed) throw new Error('Embedding provider has been disposed');
+    if (!extractorLeasePromise) {
+      extractorLeasePromise = loadSharedExtractor().then((extractor) => {
+        sharedExtractorLeases += 1;
+        return extractor;
+      });
+    }
+    return await extractorLeasePromise;
+  };
+
+  return {
+    providerType: 'local_minilm',
+    descriptor: LOCAL_MINILM_DESCRIPTOR,
+    dimension,
+    async embed(text) {
+      if (typeof text !== 'string') throw new TypeError('Embedding text must be a string');
+      const extractor = await acquire();
+      let output;
+      try {
+        output = await extractor(text, { pooling: 'mean', normalize: true });
+        if (
+          !(output.data instanceof Float32Array)
+          || output.dims?.length !== 2
+          || output.dims[0] !== 1
+          || output.dims[1] !== EMBEDDING_DIM
+        ) {
+          throw new Error(
+            `Local MiniLM returned invalid tensor shape or dtype: expected Float32[1,${EMBEDDING_DIM}]`
+          );
+        }
+        return new Float32Array(output.data);
+      } finally {
+        output?.dispose();
+      }
+    },
+    async embedBatch(texts) {
+      const results = [];
+      for (const text of texts) results.push(await this.embed(text));
+      return results;
+    },
+    embedSync() {
+      throw new Error('embedSync() is unsupported by the local neural provider; use await embed() instead');
+    },
+    async dispose() {
+      if (disposed) return;
+      disposed = true;
+      if (extractorLeasePromise) {
+        await extractorLeasePromise;
+        extractorLeasePromise = null;
+        await releaseSharedExtractor();
+      }
+    },
+  };
+}
+
+/**
+ * Creates an embedding provider. Local MiniLM is the fail-closed default; the
+ * deterministic lexical provider is available only through modelType: 'lexical'.
+ * @param {Object} [options]
+ * @param {'neural'|'lexical'|'onnx'} [options.modelType='neural']
+ * @param {Function|{run: Function, dispose?: Function}} [options.onnxRunner]
+ * @param {number} [options.dimension=384]
+ * @returns {Object}
+ */
+export function createEmbeddingProvider(options = {}) {
+  const dimension = options.dimension || EMBEDDING_DIM;
+  if (options.onnxRunner) return createAdapterProvider(options.onnxRunner, dimension);
+
+  const modelType = options.modelType || 'neural';
+  if (modelType === 'lexical') {
+    return {
+      providerType: 'lexical_hash',
+      descriptor: LEXICAL_HASH_DESCRIPTOR,
+      dimension,
+      async embed(text) {
+        return computeLexicalEmbedding(text, dimension);
+      },
+      async embedBatch(texts) {
+        return texts.map((text) => computeLexicalEmbedding(text, dimension));
+      },
+      embedSync(text) {
+        return computeLexicalEmbedding(text, dimension);
+      },
+      async dispose() {},
+    };
+  }
+  if (modelType === 'onnx') {
+    throw new Error('modelType "onnx" requires an explicit onnxRunner');
+  }
+  if (modelType !== 'neural') throw new Error(`Unsupported embedding modelType: ${modelType}`);
+  return createLocalNeuralProvider(dimension);
+}
+
+/**
+ * Cosine similarity between two float vectors.
+ * @param {Float32Array} a
+ * @param {Float32Array} b
+ * @returns {number}
+ */
+export function cosineSimilarity(a, b) {
+  if (a.length !== b.length) throw new Error(`Vector length mismatch: ${a.length} !== ${b.length}`);
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return Math.max(-1.0, Math.min(1.0, dot));
+}
+
+/**
+ * Quantizes Float32Array to Int8Array [-127, 127] for 75% memory reduction.
+ * @param {Float32Array} floatVec
+ * @returns {Int8Array}
+ */
+export function quantizeVector(floatVec) {
+  const int8 = new Int8Array(floatVec.length);
+  for (let i = 0; i < floatVec.length; i++) {
+    int8[i] = Math.round(Math.max(-1.0, Math.min(1.0, floatVec[i])) * 127);
+  }
+  return int8;
+}
+
+/**
+ * Dequantizes Int8Array back to L2-normalized Float32Array.
+ * @param {Int8Array} int8Vec
+ * @returns {Float32Array}
+ */
+export function dequantizeVector(int8Vec) {
+  const floatVec = new Float32Array(int8Vec.length);
+  let sumSq = 0;
+  for (let i = 0; i < int8Vec.length; i++) {
+    const v = int8Vec[i] / 127.0;
+    floatVec[i] = v;
+    sumSq += v * v;
+  }
+  const norm = Math.sqrt(sumSq);
+  if (norm > 1e-12) {
+    const inv = 1.0 / norm;
+    for (let i = 0; i < floatVec.length; i++) floatVec[i] *= inv;
+  }
+  return floatVec;
+}

package/packages/rag/src/hybrid.js

@@ -0,0 +1,76 @@
+// Enigma Hybrid RAG: Reciprocal Rank Fusion (RRF).
+// Blends dense vector semantic retrieval with BM25 exact lexical term matching.
+
+/**
+ * Combines dense vector search results and BM25 lexical results using Reciprocal Rank Fusion.
+ * @param {Array<{ id: string, text: string, score: number, scope: string, metadata?: Object }>} denseResults
+ * @param {Array<{ id: string, text: string, score: number, scope: string }>} bm25Results
+ * @param {Object} [options]
+ * @param {number} [options.k=60] - RRF smoothing parameter
+ * @param {number} [options.denseWeight=1.0] - Weight multiplier for dense semantic search
+ * @param {number} [options.bm25Weight=1.0] - Weight multiplier for BM25 keyword search
+ * @param {number} [options.topK=10]
+ * @returns {Array<{ id: string, text: string, score: number, denseScore: number, bm25Score: number, scope: string, metadata?: Object }>}
+ */
+export function reciprocalRankFusion(denseResults = [], bm25Results = [], options = {}) {
+  const k = options.k || 60;
+  const denseWeight = typeof options.denseWeight === 'number' ? options.denseWeight : 1.0;
+  const bm25Weight = typeof options.bm25Weight === 'number' ? options.bm25Weight : 1.0;
+  const topK = options.topK || 10;
+
+  const fusedMap = new Map();
+
+  // 1. Process Dense Vector Ranks
+  for (let rank = 0; rank < denseResults.length; rank++) {
+    const item = denseResults[rank];
+    const rrfScore = denseWeight / (k + rank + 1);
+
+    fusedMap.set(item.id, {
+      id: item.id,
+      text: item.text,
+      scope: item.scope,
+      metadata: item.metadata,
+      denseScore: item.score,
+      bm25Score: 0,
+      rrfScore,
+    });
+  }
+
+  // 2. Process BM25 Ranks
+  for (let rank = 0; rank < bm25Results.length; rank++) {
+    const item = bm25Results[rank];
+    const rrfScore = bm25Weight / (k + rank + 1);
+
+    if (fusedMap.has(item.id)) {
+      const entry = fusedMap.get(item.id);
+      entry.bm25Score = item.score;
+      entry.rrfScore += rrfScore;
+    } else {
+      fusedMap.set(item.id, {
+        id: item.id,
+        text: item.text,
+        scope: item.scope,
+        metadata: item.metadata,
+        denseScore: 0,
+        bm25Score: item.score,
+        rrfScore,
+      });
+    }
+  }
+
+  const fusedList = Array.from(fusedMap.values());
+  fusedList.sort((a, b) => b.rrfScore - a.rrfScore);
+
+  // Normalize final score relative to top match
+  const maxRrf = fusedList.length > 0 && fusedList[0].rrfScore > 0 ? fusedList[0].rrfScore : 1.0;
+
+  return fusedList.slice(0, topK).map((item) => ({
+    id: item.id,
+    text: item.text,
+    score: Math.round((item.rrfScore / maxRrf) * 1000) / 1000,
+    denseScore: Math.round(item.denseScore * 1000) / 1000,
+    bm25Score: Math.round(item.bm25Score * 1000) / 1000,
+    scope: item.scope,
+    metadata: item.metadata,
+  }));
+}

package/packages/rag/src/index.js

@@ -0,0 +1,38 @@
+// Enigma Semantic Vector Search & Hybrid RAG Engine.
+export {
+  LEXICAL_HASH_DESCRIPTOR,
+  LOCAL_MINILM_DESCRIPTOR,
+  ONNX_INTEGRATION_SPEC,
+  EMBEDDING_DIM,
+  tokenizeSubwords,
+  computeLexicalEmbedding,
+  createEmbeddingProvider,
+  verifyBundledModel,
+  cosineSimilarity,
+  quantizeVector,
+  dequantizeVector,
+} from './embeddings.js';
+
+export {
+  BM25Index,
+} from './bm25.js';
+
+export {
+  reciprocalRankFusion,
+} from './hybrid.js';
+
+export {
+  semanticReranker,
+} from './reranker.js';
+
+export {
+  EncryptedVectorStore,
+} from './vector-store.js';
+
+export {
+  RESEARCH_DISCLAIMER,
+  generateResearchOrthogonalBasis,
+  applyResearchBasisTransform,
+  createResearchBlindedQuery,
+  evaluateResearchBlindedRanking,
+} from './research.js';

package/packages/rag/src/reranker.js

@@ -0,0 +1,61 @@
+// Enigma Hybrid RAG: Semantic Candidate Reranker.
+// Calibrates retrieval scores by evaluating exact phrase matches,
+// token coverage density, and metadata importance boosts.
+
+/**
+ * Reranks candidate memories against the query for precision.
+ * @param {string} queryText
+ * @param {Array<{ id: string, text: string, score: number, scope: string, metadata?: Object }>} candidates
+ * @param {Object} [options]
+ * @param {number} [options.topK=5]
+ * @param {number} [options.phraseBonus=0.3]
+ * @param {number} [options.coverageWeight=0.25]
+ * @returns {Array<{ id: string, text: string, score: number, originalScore: number, scope: string, metadata?: Object }>}
+ */
+export function semanticReranker(queryText, candidates = [], options = {}) {
+  if (!queryText || typeof queryText !== 'string' || candidates.length === 0) {
+    return candidates.slice(0, options.topK || 5);
+  }
+
+  const topK = options.topK || 5;
+  const phraseBonus = typeof options.phraseBonus === 'number' ? options.phraseBonus : 0.3;
+  const coverageWeight = typeof options.coverageWeight === 'number' ? options.coverageWeight : 0.25;
+
+  const cleanQuery = queryText.toLowerCase().trim();
+  const queryTerms = cleanQuery.split(/[^a-zA-Z0-9_\-]/).filter(t => t.length >= 2);
+  const queryTermSet = new Set(queryTerms);
+
+  const reranked = candidates.map((cand) => {
+    const textLower = cand.text.toLowerCase();
+    let score = cand.score;
+
+    // 1. Exact phrase match boost
+    if (cleanQuery.length > 3 && textLower.includes(cleanQuery)) {
+      score += phraseBonus;
+    }
+
+    // 2. Token coverage calculation
+    if (queryTermSet.size > 0) {
+      const docTerms = new Set(textLower.split(/[^a-zA-Z0-9_\-]/).filter(t => t.length >= 2));
+      let matches = 0;
+      for (const t of queryTermSet) {
+        if (docTerms.has(t)) matches++;
+      }
+      const coverage = matches / queryTermSet.size;
+      score += coverage * coverageWeight;
+    }
+
+    // 3. Metadata importance multiplier
+    const importance = cand.metadata?.importance || cand.importance || 1.0;
+    score = score * Math.min(2.0, Math.max(0.5, importance));
+
+    return {
+      ...cand,
+      originalScore: cand.score,
+      score: Math.round(score * 1000) / 1000,
+    };
+  });
+
+  reranked.sort((a, b) => b.score - a.score);
+  return reranked.slice(0, topK);
+}