mumpix 1.0.20 → 1.1.3

package/scripts/test-semantic.cjs

@@ -0,0 +1,411 @@
+"use strict";
+
+const assert = require("assert");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { spawn } = require("child_process");
+const { Mumpix } = require("../src/index");
+
+const DIM = 16;
+const FOLD = {
+  realtime: "websocket",
+  socket: "websocket",
+  login: "auth",
+  check: "middleware",
+  skip: "bypass",
+};
+
+function fnv1a(text) {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < text.length; i += 1) {
+    hash ^= text.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return hash >>> 0;
+}
+
+class TestEmbedder {
+  constructor(id = "test-embedder-v1") {
+    this.id = id;
+    this.dim = DIM;
+  }
+
+  async embed(texts) {
+    return texts.map((text) => {
+      const vector = new Float32Array(DIM);
+      for (const raw of String(text).toLowerCase().split(/[^a-z0-9]+/).filter(Boolean)) {
+        const token = FOLD[raw] || raw;
+        const hash = fnv1a(token);
+        vector[hash % DIM] += (hash >>> 16) & 1 ? 1 : -1;
+      }
+      const norm = Math.hypot(...vector) || 1;
+      for (let i = 0; i < DIM; i += 1) vector[i] /= norm;
+      return vector;
+    });
+  }
+}
+
+function clean(filePath) {
+  for (const suffix of ["", ".wal", ".lock", ".idx", ".vectors.json", ".vectors.jsonl"]) {
+    fs.rmSync(`${filePath}${suffix}`, { force: true, recursive: true });
+  }
+}
+
+async function testBasicSemanticRecall() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "mumpix-semantic-test-"));
+  const filePath = path.join(dir, "semantic.mumpix");
+  clean(filePath);
+
+  const db = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    await db.remember("websocket upgrade requests bypass express middleware auth");
+    assert.strictEqual(
+      await db.recall("realtime socket connections skip the login check"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+    const liveHits = await db.recallMany("realtime socket connections skip the login check", 1);
+    assert.strictEqual(liveHits.length, 1);
+    assert.strictEqual(liveHits[0].id, 1);
+    assert.strictEqual(
+      liveHits[0].content,
+      "websocket upgrade requests bypass express middleware auth"
+    );
+    assert.strictEqual(Number.isFinite(liveHits[0].score), true);
+    assert.strictEqual(await db.recall("purple elephant tax return kazoo"), null);
+  } finally {
+    await db.close();
+  }
+
+  const sidecarPath = `${filePath}.vectors.json`;
+  const sidecar = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  assert.strictEqual(sidecar.embedder_id, "test-embedder-v1");
+  assert.strictEqual(sidecar.entries.length, 1);
+  assert.strictEqual(sidecar.entries[0].seq, 1);
+  assert.strictEqual(sidecar.entries[0].id, 1);
+  assert.strictEqual(sidecar.entries[0].record_id, 1);
+  assert.strictEqual(fs.existsSync(`${sidecarPath}l`), false);
+
+  const reopened = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    assert.strictEqual(
+      await reopened.recall("realtime socket connections skip the login check"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+    const reopenedHits = await reopened.recallMany(
+      "realtime socket connections skip the login check",
+      1
+    );
+    assert.strictEqual(reopenedHits.length, 1);
+    assert.strictEqual(reopenedHits[0].id, 1);
+  } finally {
+    await reopened.close();
+  }
+
+  const changed = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder("test-embedder-v2"),
+    semanticFloor: 0.35,
+  });
+  try {
+    assert.strictEqual(
+      await changed.recall("realtime socket connections skip the login check"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+  } finally {
+    await changed.close();
+  }
+
+  const changedSidecar = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  assert.strictEqual(changedSidecar.embedder_id, "test-embedder-v2");
+  assert.strictEqual(changedSidecar.entries.length, 1);
+
+  let missingError = null;
+  try {
+    await Mumpix.open(path.join(dir, "missing-embedder.mumpix"), { recall: "semantic" });
+  } catch (err) {
+    missingError = err;
+  }
+  assert.ok(missingError, "semantic recall without an embedder must fail loudly");
+  assert.match(String(missingError.message), /mumpix-embed/);
+
+  clean(filePath);
+  fs.rmSync(dir, { force: true, recursive: true });
+}
+
+async function testSemanticReconcileRepairsMissingVectors() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "mumpix-semantic-reconcile-"));
+  const filePath = path.join(dir, "semantic.mumpix");
+  clean(filePath);
+
+  const db = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    await db.remember("websocket upgrade requests bypass express middleware auth");
+  } finally {
+    await db.close();
+  }
+
+  const sidecarPath = `${filePath}.vectors.json`;
+  const sidecar = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  sidecar.entries = [];
+  fs.writeFileSync(sidecarPath, JSON.stringify(sidecar), "utf8");
+
+  const reopened = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    assert.strictEqual(
+      await reopened.recall("realtime socket connections skip the login check"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+  } finally {
+    await reopened.close();
+  }
+
+  const repaired = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  assert.strictEqual(repaired.entries.length, 1);
+  assert.strictEqual(repaired.entries[0].seq, 1);
+
+  clean(filePath);
+  fs.rmSync(dir, { force: true, recursive: true });
+}
+
+async function testSemanticReconcileDropsTornJournalTail() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "mumpix-semantic-torn-"));
+  const filePath = path.join(dir, "semantic.mumpix");
+  clean(filePath);
+
+  const db = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    await db.remember("websocket upgrade requests bypass express middleware auth");
+  } finally {
+    await db.close();
+  }
+
+  const sidecarPath = `${filePath}.vectors.json`;
+  const sidecar = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  sidecar.entries = [];
+  fs.writeFileSync(sidecarPath, JSON.stringify(sidecar), "utf8");
+  fs.writeFileSync(`${sidecarPath}l`, "{\"op\":\"upsert\",\"entry\":", "utf8");
+
+  const reopened = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    assert.strictEqual(
+      await reopened.recall("realtime socket connections skip the login check"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+  } finally {
+    await reopened.close();
+  }
+
+  const repaired = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  assert.strictEqual(repaired.entries.length, 1);
+  assert.strictEqual(fs.existsSync(`${sidecarPath}l`), false);
+
+  clean(filePath);
+  fs.rmSync(dir, { force: true, recursive: true });
+}
+
+async function testPlainOpenDefersSemanticReconcile() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "mumpix-semantic-defer-"));
+  const filePath = path.join(dir, "semantic.mumpix");
+  clean(filePath);
+
+  const db = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    await db.remember("websocket upgrade requests bypass express middleware auth");
+  } finally {
+    await db.close();
+  }
+
+  const sidecarPath = `${filePath}.vectors.json`;
+  const sidecar = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  sidecar.entries = [];
+  fs.writeFileSync(sidecarPath, JSON.stringify(sidecar), "utf8");
+
+  const plain = await Mumpix.open(filePath);
+  try {
+    assert.strictEqual(
+      await plain.recall("websocket auth middleware"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+  } finally {
+    await plain.close();
+  }
+
+  const deferred = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  assert.strictEqual(deferred.entries.length, 0);
+
+  const semantic = await Mumpix.open(filePath, {
+    recall: "semantic",
+    embedder: new TestEmbedder(),
+    semanticFloor: 0.35,
+  });
+  try {
+    assert.strictEqual(
+      await semantic.recall("realtime socket connections skip the login check"),
+      "websocket upgrade requests bypass express middleware auth"
+    );
+  } finally {
+    await semantic.close();
+  }
+
+  const repaired = JSON.parse(fs.readFileSync(sidecarPath, "utf8"));
+  assert.strictEqual(repaired.entries.length, 1);
+  assert.strictEqual(repaired.entries[0].seq, 1);
+
+  clean(filePath);
+  fs.rmSync(dir, { force: true, recursive: true });
+}
+
+async function testSemanticSigkillReconcilesSidecar() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "mumpix-semantic-sigkill-"));
+  const filePath = path.join(dir, "semantic.mumpix");
+  clean(filePath);
+
+  const childCode = `
+    const { Mumpix } = require(${JSON.stringify(path.resolve(__dirname, "../src/index"))});
+    class SlowEmbedder {
+      constructor() { this.id = "slow-test-embedder-v1"; this.dim = ${DIM}; }
+      async embed(texts) {
+        await new Promise((resolve) => setTimeout(resolve, 25));
+        return texts.map((text) => {
+          const vector = new Float32Array(${DIM});
+          for (const raw of String(text).toLowerCase().split(/[^a-z0-9]+/).filter(Boolean)) {
+            const token = ${JSON.stringify(FOLD)}[raw] || raw;
+            let hash = 0x811c9dc5;
+            for (let i = 0; i < token.length; i += 1) {
+              hash ^= token.charCodeAt(i);
+              hash = Math.imul(hash, 0x01000193);
+            }
+            vector[(hash >>> 0) % ${DIM}] += (hash >>> 16) & 1 ? 1 : -1;
+          }
+          const norm = Math.hypot(...vector) || 1;
+          for (let i = 0; i < ${DIM}; i += 1) vector[i] /= norm;
+          return vector;
+        });
+      }
+    }
+    (async () => {
+      const db = await Mumpix.open(${JSON.stringify(filePath)}, {
+        recall: "semantic",
+        embedder: new SlowEmbedder(),
+        semanticFloor: 0.35,
+      });
+      for (let i = 0; i < 200; i += 1) {
+        await db.remember("websocket upgrade requests bypass express middleware auth " + i);
+        if (i === 10 && process.send) process.send({ ready: true });
+      }
+      if (process.send) process.send({ done: true });
+      setInterval(() => {}, 1000);
+    })().catch((err) => {
+      if (process.send) process.send({ error: err.message });
+      process.exit(1);
+    });
+  `;
+
+  const child = spawn(process.execPath, ["-e", childCode], {
+    stdio: ["ignore", "ignore", "ignore", "ipc"],
+  });
+
+  try {
+    await new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error("semantic child did not start")), 5000);
+      child.on("message", (message) => {
+        if (message && message.error) {
+          clearTimeout(timeout);
+          reject(new Error(message.error));
+        } else if (message && message.ready) {
+          clearTimeout(timeout);
+          resolve();
+        }
+      });
+      child.once("exit", (code, signal) => {
+        clearTimeout(timeout);
+        reject(new Error(`child exited before ready: code=${code} signal=${signal}`));
+      });
+    });
+
+    child.kill("SIGKILL");
+    await new Promise((resolve) => child.once("exit", resolve));
+
+    const fileRecordsBeforeReplay = fs
+      .readFileSync(filePath, "utf8")
+      .split("\n")
+      .filter((line) => line.trim()).length - 1;
+    assert(fileRecordsBeforeReplay > 0, "SIGKILL fixture should leave records in the ledger");
+
+    const reopened = await Mumpix.open(filePath, {
+      recall: "semantic",
+      embedder: new TestEmbedder("slow-test-embedder-v1"),
+      semanticFloor: 0.35,
+    });
+    try {
+      const records = await reopened.list();
+      assert.ok(
+        records.length >= fileRecordsBeforeReplay,
+        "reopen should preserve main-file records and may replay additional durable WAL records"
+      );
+      const recalled = await reopened.recall("realtime socket connections skip the login check");
+      assert.ok(
+        records.some((record) => record.content === recalled),
+        "semantic recall should return a surviving ledger record after SIGKILL"
+      );
+    } finally {
+      await reopened.close();
+    }
+
+    const sidecar = JSON.parse(fs.readFileSync(`${filePath}.vectors.json`, "utf8"));
+    const reopenedCount = fs
+      .readFileSync(filePath, "utf8")
+      .split("\n")
+      .filter((line) => line.trim()).length - 1;
+    assert.strictEqual(sidecar.entries.length, reopenedCount);
+    assert.strictEqual(
+      Math.max(...sidecar.entries.map((entry) => entry.seq)),
+      reopenedCount
+    );
+  } finally {
+    if (!child.killed) child.kill("SIGKILL");
+    clean(filePath);
+    fs.rmSync(dir, { force: true, recursive: true });
+  }
+}
+
+(async () => {
+  await testBasicSemanticRecall();
+  await testSemanticReconcileRepairsMissingVectors();
+  await testSemanticReconcileDropsTornJournalTail();
+  await testPlainOpenDefersSemanticReconcile();
+  await testSemanticSigkillReconcilesSidecar();
+  console.log("semantic recall tests passed");
+})().catch((err) => {
+  console.error(err && err.stack ? err.stack : err);
+  process.exit(1);
+});

package/src/brp/index.js

@@ -0,0 +1 @@
+"use strict";const e=require("crypto"),r=["active","probationary","archived","fossilized"],t=["DO","PREFER"],n=["fresh_promotion","refined_from","trigger_refresh"],o=["navigate","read-dom","input-text","click-element","extract-kv","wait-for-state","scroll-page","compress-context","read-local-state"];function i(r="brp"){const t=e.randomUUID?e.randomUUID():e.randomBytes(16).toString("hex");return`${r}_${String(t).replace(/-/g,"").slice(0,16)}`}function a(e){return String(e||"").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean)}function s(e){if(e&&"object"==typeof e)return e;if("string"!=typeof e||!e.trim())throw new Error("executable_payload is empty or null.");try{return JSON.parse(e)}catch{throw new Error("executable_payload failed JSON parse.")}}class c{constructor(e=o){this._verbs=new Map;for(const r of e)this.register(r)}register(e,r=null){const t=String(e||"").trim();if(!t)throw new Error("verb name is required");return this._verbs.set(t,r||!0),this}has(e){return this._verbs.has(String(e||"").trim())}list(){return Array.from(this._verbs.keys()).sort()}}function u(e,r={}){const n=r.verbRegistry||new c(r.verbs||o),i=s(e);if(!i||"object"!=typeof i)throw new Error("executable_payload must parse to an object.");if(!t.includes(i.intent))throw new Error("executable_payload intent must be DO or PREFER.");if("DO"===i.intent){if(!Array.isArray(i.sequence)||0===i.sequence.length)throw new Error("DO payload has empty sequence.");for(const e of i.sequence){if(!e||"object"!=typeof e)throw new Error("DO payload step must be an object.");if(!n.has(e.verb))throw new Error(`Unknown verb: ${e.verb}`);if(null!=e.args&&("object"!=typeof e.args||Array.isArray(e.args)))throw new Error(`DO payload step args must be an object for verb: ${e.verb}`)}}if("PREFER"===i.intent){if("string"!=typeof i.injected_context||!i.injected_context.trim())throw new Error("PREFER payload requires injected_context.");if(null!=i.negative_constraints&&!Array.isArray(i.negative_constraints))throw new Error("PREFER negative_constraints must be an array when present.")}return i}function g(e,t={}){try{if(!e||"object"!=typeof e)throw new Error("BRP candidate must be an object.");if(!e.domain)throw new Error("BRP must declare a domain.");if(!e.trigger_signature)throw new Error("BRP must declare a trigger_signature.");const n=u(e.executable_payload,t);if(e.status&&!r.includes(e.status))throw new Error(`Unknown BRP status: ${e.status}`);return{ok:!0,payload:n,payload_type:n.intent,warnings:[]}}catch(e){return{ok:!1,error:e.message,warnings:[]}}}function _(e,r={}){const t=r.domain?String(r.domain):null,n=r.statuses||["active","probationary"];return e.filter(e=>(!t||e.domain===t)&&(!(n&&!n.includes(e.status))&&(!r.payload_type||e.payload_type===r.payload_type)))}function d(e,r){const t=new Set(a(r));if(!t.size)return 0;const n=a(`${e.trigger_signature} ${e.domain} ${e.payload_type}`);let o=0;for(const e of n)t.has(e)&&(o+=1);return o+(Number(e.trigger_confidence||0)+Number(e.sequence_confidence||0))/2}module.exports={CORE_ACTION_VERBS:o,GENE_STATUSES:r,PAYLOAD_TYPES:t,PROVENANCE_TYPES:n,VerbRegistry:c,parseExecutablePayload:s,validatePayload:u,validateBRP:g,normalizeBRP:function(e,r={}){const t=g(e,r);if(!t.ok)throw new Error(t.error);const o=(new Date).toISOString(),a=t.payload;return{brp_id:e.brp_id||i("brp"),domain:String(e.domain),status:e.status||"active",trigger_signature:String(e.trigger_signature),trigger_embedding:Array.isArray(e.trigger_embedding)?e.trigger_embedding.map(Number):[],preconditions:e.preconditions&&"object"==typeof e.preconditions?{...e.preconditions}:{},executable_payload:"string"==typeof e.executable_payload?e.executable_payload:JSON.stringify(e.executable_payload),payload_type:a.intent,sequence_execution_count:Number(e.sequence_execution_count||0),sequence_confidence:Number.isFinite(Number(e.sequence_confidence))?Number(e.sequence_confidence):1,trigger_match_count:Number(e.trigger_match_count||0),trigger_confidence:Number.isFinite(Number(e.trigger_confidence))?Number(e.trigger_confidence):.5,trigger_version:Number(e.trigger_version||1),break_even_crossed:Boolean(e.break_even_crossed),created_at:e.created_at||o,status_changed_at:e.status_changed_at||o,...e.retirement_reason?{retirement_reason:String(e.retirement_reason)}:{},provenance:n.includes(e.provenance)?e.provenance:"fresh_promotion"}},updateSplitConfidence:function(e,r={}){const t={...e};r.triggerMatched&&(t.trigger_match_count=Number(t.trigger_match_count||0)+1,t.trigger_confidence=Math.min(1,Number(t.trigger_confidence||0)+.05)),r.triggerDrift&&(t.trigger_confidence=Math.max(0,Number(t.trigger_confidence||0)-.15),t.trigger_version=Number(t.trigger_version||1)+1),r.sequenceExecuted&&(t.sequence_execution_count=Number(t.sequence_execution_count||0)+1,t.sequence_confidence=Math.min(1,Number(t.sequence_confidence||0)+.03)),r.sequenceFailed&&(t.sequence_confidence=Math.max(0,Number(t.sequence_confidence||0)-.2));const n=Number.isFinite(Number(r.breakEvenThreshold))?Number(r.breakEvenThreshold):3;return t.break_even_crossed=Number(t.sequence_execution_count||0)>=n,t},transitionStatus:function(e,t,n=""){if(!r.includes(t))throw new Error(`Unknown BRP status: ${t}`);return{...e,status:t,status_changed_at:(new Date).toISOString(),...n?{retirement_reason:String(n)}:{}}},structuralPrefilter:_,twoStageMatch:function(e,r={},t={}){const n=_(e,r),o=r.text||r.trigger_signature||"",i=Number(t.k||r.k||5);return n.map(e=>({brp:e,score:d(e,o)})).sort((e,r)=>r.score-e.score).slice(0,i)}};

package/src/collapse/index.js

@@ -0,0 +1 @@
+"use strict";function e(e){return"number"==typeof e?e:e&&"object"==typeof e&&Number(e.token_cost??e.tokens??e.cost??0)||0}function r(r=[]){if(!Array.isArray(r))throw new TypeError("graphCost() expects an array of stages or costs");return r.reduce((r,t)=>r+e(t),0)}function t(e=[],t={}){const o=Number.isFinite(Number(t.cycleCost))?Number(t.cycleCost):r(e),s=Number.isFinite(Number(t.retrievalCost))?Number(t.retrievalCost):150;return Math.max(0,o-s)}function o(e,r){const t=Number(e),o=Number(r);if(!Number.isFinite(t)||t<0)throw new TypeError("promotionCost must be a non-negative number");if(!Number.isFinite(o)||o<=0)throw new TypeError("cycleCost must be a positive number");return t/o}function s(e,r){const t=Number(e),o=Number(r);return!Number.isFinite(t)||!Number.isFinite(o)||o<=0?null:(t-o)/o}module.exports={DEFAULT_RETRIEVAL_COST:150,stageCost:e,graphCost:r,collapseSavings:t,breakEvenReuseThreshold:o,beta:s,analyzeCollapse:function(e={}){const i=e.stages||[],n=Number.isFinite(Number(e.cycleCost))?Number(e.cycleCost):r(i),u=Number.isFinite(Number(e.retrievalCost))?Number(e.retrievalCost):150,b=Number.isFinite(Number(e.promotionCost))?Number(e.promotionCost):3*n,m=o(b,n||1),a=t([],{cycleCost:n,retrievalCost:u}),c=Number(e.hits||1);return{cycleCost:n,promotionCost:b,retrievalCost:u,savingsPerHit:a,hits:c,totalSavings:a*c,breakEvenReuseThreshold:m,beta:s(b,n),breakEvenCrossed:c>=m}}};