mumpix 1.0.19 → 1.0.29

package/src/integrations/vector-sidecar.js

@@ -0,0 +1,325 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+function defaultVectorPath(db) {
+  if (!db || !db.path) {
+    throw new Error("Mumpix vector adapters require an opened MumpixDB instance with a path.");
+  }
+  return `${db.path}.vectors.json`;
+}
+
+function emptyData() {
+  return {
+    version: 1,
+    embeddingDict: {},
+    textIdToRecordId: {},
+    textIdToRefDocId: {},
+    metadataDict: {},
+    textDict: {},
+  };
+}
+
+function loadData(vectorPath) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(vectorPath, "utf8"));
+    return { ...emptyData(), ...(parsed && typeof parsed === "object" ? parsed : {}) };
+  } catch {
+    return emptyData();
+  }
+}
+
+function saveData(vectorPath, data) {
+  fs.mkdirSync(path.dirname(vectorPath), { recursive: true });
+  const tmp = `${vectorPath}.tmp`;
+  fs.writeFileSync(tmp, JSON.stringify({ ...emptyData(), ...data, version: 1 }, null, 2), "utf8");
+  fs.renameSync(tmp, vectorPath);
+}
+
+function dot(left, right) {
+  const length = Math.min(left.length, right.length);
+  let score = 0;
+  for (let i = 0; i < length; i += 1) score += Number(left[i] || 0) * Number(right[i] || 0);
+  return score;
+}
+
+function magnitude(vector) {
+  return Math.sqrt(dot(vector, vector));
+}
+
+function cosine(left, right) {
+  if (!(Array.isArray(left) && Array.isArray(right) && left.length && right.length)) return 0;
+  const denom = magnitude(left) * magnitude(right);
+  return denom ? dot(left, right) / denom : 0;
+}
+
+function filterMatches(metadata, filters) {
+  if (!filters || !Array.isArray(filters.filters)) return true;
+  const condition = String(filters.condition || "and").toLowerCase();
+  const checks = filters.filters.map((filter) => {
+    const actual = metadata ? metadata[filter.key] : undefined;
+    const expected = filter.value;
+    const operator = filter.operator || "==";
+    if (operator === "is_empty") {
+      return actual == null || actual === "" || (Array.isArray(actual) && actual.length === 0);
+    }
+    if (actual === undefined) return false;
+    if (operator === "==" || operator === "eq") return actual === expected;
+    if (operator === "!=" || operator === "ne") return actual !== expected;
+    if (operator === "in") return Array.isArray(expected) && expected.includes(actual);
+    if (operator === "nin") return Array.isArray(expected) && !expected.includes(actual);
+    if (operator === "any") return Array.isArray(actual) && Array.isArray(expected) && expected.some((value) => actual.includes(value));
+    if (operator === "all") return Array.isArray(actual) && Array.isArray(expected) && expected.every((value) => actual.includes(value));
+    if (operator === "contains") return Array.isArray(actual) && actual.includes(expected);
+    if (operator === "text_match") return String(actual).includes(String(expected));
+    if (operator === ">") return Number(actual) > Number(expected);
+    if (operator === "<") return Number(actual) < Number(expected);
+    if (operator === ">=") return Number(actual) >= Number(expected);
+    if (operator === "<=") return Number(actual) <= Number(expected);
+    return false;
+  });
+  return condition === "or" ? checks.some(Boolean) : checks.every(Boolean);
+}
+
+function topKPush(top, hit, k) {
+  if (k <= 0) return;
+  if (top.length < k) {
+    top.push(hit);
+    if (top.length === k) top.sort((a, b) => a.score - b.score);
+    return;
+  }
+  if (hit.score <= top[0].score) return;
+  top[0] = hit;
+  let index = 0;
+  while (index + 1 < top.length && top[index].score > top[index + 1].score) {
+    const next = top[index + 1];
+    top[index + 1] = top[index];
+    top[index] = next;
+    index += 1;
+  }
+}
+
+function intersectSets(left, right) {
+  if (!left) return new Set(right);
+  const out = new Set();
+  for (const value of left) {
+    if (right.has(value)) out.add(value);
+  }
+  return out;
+}
+
+function metadataValueKey(value) {
+  return `${typeof value}:${String(value)}`;
+}
+
+function equalityCandidateSet(index, filters) {
+  if (!filters || !Array.isArray(filters.filters)) return null;
+  if (String(filters.condition || "and").toLowerCase() !== "and") return null;
+  let candidates = null;
+  for (const filter of filters.filters) {
+    const operator = filter.operator || "==";
+    if (operator !== "==" && operator !== "eq") continue;
+    const byValue = index.metadataIndexes.get(String(filter.key));
+    if (!byValue) return new Set();
+    const ids = byValue.get(metadataValueKey(filter.value));
+    if (!ids) return new Set();
+    candidates = intersectSets(candidates, ids);
+    if (candidates.size === 0) return candidates;
+  }
+  return candidates;
+}
+
+function buildPackedIndex(data) {
+  const ids = [];
+  const vectors = [];
+  let dims = 0;
+
+  for (const [id, vector] of Object.entries(data.embeddingDict || {})) {
+    if (!Array.isArray(vector) || vector.length === 0) continue;
+    if (!dims) dims = vector.length;
+    if (vector.length !== dims) continue;
+    ids.push(String(id));
+    vectors.push(vector);
+  }
+
+  const packed = new Float32Array(ids.length * dims);
+  const norms = new Float32Array(ids.length);
+  const idToOffset = new Map();
+
+  for (let row = 0; row < vectors.length; row += 1) {
+    const vector = vectors[row];
+    let norm = 0;
+    const base = row * dims;
+    for (let col = 0; col < dims; col += 1) {
+      const value = Number(vector[col] || 0);
+      packed[base + col] = value;
+      norm += value * value;
+    }
+    norms[row] = Math.sqrt(norm) || 0;
+    idToOffset.set(ids[row], row);
+  }
+
+  const metadataIndexes = new Map();
+  for (const [id, metadata] of Object.entries(data.metadataDict || {})) {
+    if (!idToOffset.has(String(id)) || !metadata || typeof metadata !== "object") continue;
+    for (const [key, value] of Object.entries(metadata)) {
+      if (value == null || typeof value === "object") continue;
+      const field = String(key);
+      if (!metadataIndexes.has(field)) metadataIndexes.set(field, new Map());
+      const byValue = metadataIndexes.get(field);
+      const valueKey = metadataValueKey(value);
+      if (!byValue.has(valueKey)) byValue.set(valueKey, new Set());
+      byValue.get(valueKey).add(String(id));
+    }
+  }
+
+  return { ids, dims, packed, norms, idToOffset, metadataIndexes };
+}
+
+function vectorNorm(query) {
+  let norm = 0;
+  for (let i = 0; i < query.length; i += 1) {
+    const value = Number(query[i] || 0);
+    norm += value * value;
+  }
+  return Math.sqrt(norm) || 0;
+}
+
+function packedCosine(index, row, query, queryNorm) {
+  if (!index.dims || !queryNorm || !index.norms[row]) return 0;
+  const base = row * index.dims;
+  let score = 0;
+  for (let col = 0; col < index.dims; col += 1) {
+    score += index.packed[base + col] * Number(query[col] || 0);
+  }
+  return score / (index.norms[row] * queryNorm);
+}
+
+function candidateIdsFromOptions(index, options) {
+  let candidates = options.docIds ? new Set(options.docIds.map(String)) : null;
+  const equalitySet = equalityCandidateSet(index, options.filters);
+  if (equalitySet) candidates = intersectSets(candidates, equalitySet);
+  return candidates;
+}
+
+function queryDefault(data, index, query, k, options = {}) {
+  const limit = Math.max(0, Number(k) || 0);
+  if (!limit) return [];
+
+  const candidates = candidateIdsFromOptions(index, options);
+  const queryNorm = vectorNorm(query);
+  const top = [];
+  const ids = candidates ? Array.from(candidates) : index.ids;
+
+  for (const id of ids) {
+    const row = index.idToOffset.get(String(id));
+    if (row == null) continue;
+    const metadata = data.metadataDict[id] || {};
+    if (!filterMatches(metadata, options.filters)) continue;
+    topKPush(top, { id, score: packedCosine(index, row, query, queryNorm) }, limit);
+  }
+
+  return top.sort((a, b) => b.score - a.score);
+}
+
+function queryMmr(data, index, query, k, options = {}) {
+  const lambda = Number.isFinite(options.lambda) ? options.lambda : 0.5;
+  const pool = queryDefault(data, index, query, Math.max(4 * k, k), options);
+  const selected = [];
+  while (selected.length < k && pool.length) {
+    let bestIndex = 0;
+    let bestScore = -Infinity;
+    for (let i = 0; i < pool.length; i += 1) {
+      const candidate = pool[i];
+      const candidateVector = data.embeddingDict[candidate.id] || [];
+      const diversity = selected.length
+        ? Math.max(...selected.map((hit) => cosine(candidateVector, data.embeddingDict[hit.id] || [])))
+        : 0;
+      const score = lambda * candidate.score - (1 - lambda) * diversity;
+      if (score > bestScore) {
+        bestScore = score;
+        bestIndex = i;
+      }
+    }
+    selected.push(pool.splice(bestIndex, 1)[0]);
+  }
+  return selected;
+}
+
+class MumpixVectorSidecar {
+  constructor(db, options = {}) {
+    this.db = db;
+    this.vectorPath = options.vectorPath || defaultVectorPath(db);
+    this.data = options.data || loadData(this.vectorPath);
+    this._packedIndex = null;
+  }
+
+  persist() {
+    saveData(this.vectorPath, this.data);
+  }
+
+  _invalidateIndex() {
+    this._packedIndex = null;
+  }
+
+  _index() {
+    if (!this._packedIndex) this._packedIndex = buildPackedIndex(this.data);
+    return this._packedIndex;
+  }
+
+  get(id) {
+    return this.data.embeddingDict[String(id)] || null;
+  }
+
+  async addItem({ id, text, embedding, metadata = {}, refDocId = null }) {
+    if (!Array.isArray(embedding)) {
+      throw new Error("Mumpix vector adapters require numeric embeddings.");
+    }
+    const record = await this.db.remember(text || metadata.text || String(id));
+    const textId = String(id || record.id);
+    this.data.embeddingDict[textId] = embedding.map(Number);
+    this.data.textIdToRecordId[textId] = String(record.id);
+    this.data.textIdToRefDocId[textId] = String(refDocId || metadata.refDocId || record.id);
+    this.data.metadataDict[textId] = { ...metadata, mumpixRecordId: record.id };
+    this.data.textDict[textId] = text || metadata.text || "";
+    this._invalidateIndex();
+    this.persist();
+    return textId;
+  }
+
+  deleteRefDoc(refDocId) {
+    const target = String(refDocId);
+    for (const [textId, storedRefDocId] of Object.entries(this.data.textIdToRefDocId)) {
+      if (String(storedRefDocId) !== target && String(textId) !== target) continue;
+      delete this.data.embeddingDict[textId];
+      delete this.data.textIdToRecordId[textId];
+      delete this.data.textIdToRefDocId[textId];
+      delete this.data.metadataDict[textId];
+      delete this.data.textDict[textId];
+    }
+    this._invalidateIndex();
+    this.persist();
+  }
+
+  query(query, k, options = {}) {
+    const index = this._index();
+    const hits =
+      String(options.mode || "default").toLowerCase() === "mmr"
+        ? queryMmr(this.data, index, query, k, options)
+        : queryDefault(this.data, index, query, k, options);
+    return {
+      ids: hits.map((hit) => hit.id),
+      similarities: hits.map((hit) => hit.score),
+      records: hits.map((hit) => ({
+        id: hit.id,
+        score: hit.score,
+        text: this.data.textDict[hit.id] || "",
+        metadata: this.data.metadataDict[hit.id] || {},
+        embedding: this.data.embeddingDict[hit.id] || [],
+      })),
+    };
+  }
+}
+
+module.exports = { MumpixVectorSidecar, cosine };

package/src/rlp/index.js

@@ -0,0 +1 @@
+"use strict";const e=[{id:"promotion",name:"Promotion mechanism",description:"Records successful patterns as genes with evidence thresholds."},{id:"retrieval",name:"Retrieval mechanism",description:"Queries the gene library before LLM fallback."},{id:"domain_routing",name:"Domain routing",description:"Scopes retrieval to prevent cross-domain monopoly."},{id:"retrieval_pathway_repair",name:"Retrieval pathway repair",description:"Ensures promoted genes are reachable and their actions are executable."},{id:"decay_calibration",name:"Decay calibration",description:"Keeps confidence decay thresholds above the empirical natural concentration baseline."}],a=[["gap_1",1,"Promotion without retrieval","Genes are promoted but never queried."],["gap_2",2,"Universal policy collapse","A single gene monopolizes all decisions."],["gap_3",3,"Confidence asymmetry","High-confidence universal genes crowd out specialized genes."],["gap_4",4,"Ranking exclusion","Non-dominant genes are absent from the top-k candidate set."],["gap_5",5,"Specification-implementation gap","Domain filters are accepted but not enforced."],["gap_6",6,"Promotion-retrieval orthogonality","Promotion does not guarantee utilization."],["gap_7",7,"Zero-learning fixed point","Exploration set to zero prevents new promotions."],["gap_8",8,"Bootstrap phase stall","No specialized genes exist until enough LLM observations accumulate."],["gap_9",9,"Bypass pathway promotions","Promotions enter a ledger path that does not participate in retrieval."],["gap_10",10,"Action Deserialization Failure","Genes are credited as executed while action fields are empty or unparseable."],["gap_11",11,"Confidence Decay Miscalibration","Anti-monopoly decay is set at or below the natural concentration baseline."],["gap_12",12,"Execution State Contamination","A gene sequence fails mid-execution and leaves contaminated environment state."],["gap_13",13,"Trigger Signature Drift","The environment evolves so a trigger no longer denotes the same condition."],["gap_14",14,"Trigger-Sequence Divergence","Trigger revalidations diverge too far from the original proven sequence."]].map(([e,a,t,n])=>({id:e,number:a,name:t,description:n}));function t(e){const t=String(e).toLowerCase();return a.find(e=>String(e.number)===t||e.id===t||e.id===`gap_${t}`)||null}module.exports={REQUIRED_MECHANISMS:e,FAILURE_MODES:a,ARCHITECTURAL_LAWS:[{number:1,name:"Domain Constraint",statement:"Any globally-ranked behavioral retrieval system without domain constraints will converge to a single-policy attractor state."},{number:2,name:"Decay Calibration",statement:"Any confidence-decay mechanism whose trigger lies at or below the domain empirical concentration baseline will re-instantiate universal policy collapse."},{number:3,name:"Execution State Isolation",statement:"Any behavioral execution system without execution state isolation will converge to contaminated context as a stable failure mode."}],gap:t,validateActionBoundary:function(e){if(!e||"object"!=typeof e)return{ok:!1,gap:t(10),error:"record must be an object"};const a=e.executable_payload||e.action||e.payload;if(!a)return{ok:!1,gap:t(10),error:"action payload is empty"};try{const e="string"==typeof a?JSON.parse(a):a;if(!e||"object"!=typeof e)throw new Error("payload must be an object");return"DO"!==e.intent||Array.isArray(e.sequence)&&0!==e.sequence.length?{ok:!0,payload:e}:{ok:!1,gap:t(10),error:"DO payload has no executable sequence"}}catch(e){return{ok:!1,gap:t(10),error:e.message}}},mechanismChecklist:function(a={}){const t=new Set(Object.entries(a).filter(([,e])=>Boolean(e)).map(([e])=>e));return e.map(e=>({...e,present:t.has(e.id)}))}};