@xmbl/state-machine 0.1.4 → 0.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xmbl/state-machine",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "type": "module",
   "main": "index.js",
   "exports": {

package/src/rehydration.test.mjs

@@ -0,0 +1,94 @@
+// THE THREE DEFECTS THAT KEPT NODES OFF A SHARED ROOT, each asserted by the symptom the fleet showed,
+// not by the mechanism. All three were measured live on 2026-09-15 before being fixed here.
+import { test } from 'node:test';
+import assert from 'node:assert';
+import { rmSync } from 'node:fs';
+import { StateMachine } from './state-machine.js';
+
+const ZERO = '0'.repeat(64);
+const anchors = (n) => Array.from({ length: n }, (_, i) => ({
+  event: 'task.created', hash: `h${String(i).padStart(4, '0')}`.padEnd(12, '0'), ts: 1000 + i,
+}));
+const blockFor = (a, tag) => ({ id: `blk-${tag}-${Math.random().toString(16).slice(2)}`,
+  tx: { type: 'anchor', event: a.event, hash: a.hash, ts: a.ts } });
+const countDiffRows = async (sm) => { let n = 0; for await (const [] of sm.db.iterator({ gt: 'diff:', lt: 'diff:\xFF' })) n++; return n; };
+
+async function open(dir) { const sm = new StateMachine({ dbPath: dir }); await sm.ready(); return sm; }
+
+test('a restarted node publishes a REAL root, not 64 zeros, from the state: keyspace alone', async () => {
+  const dir = `/tmp/xvsm-rehydrate-${process.pid}-a`;
+  rmSync(dir, { recursive: true, force: true });
+  let sm = await open(dir);
+  for (const a of anchors(30)) await sm._handleLedgerBlock(blockFor(a, 'x'));
+  const before = sm.stateTree.getRoot();
+  assert.notStrictEqual(before, ZERO);
+  await sm.db.close();
+
+  // Reopen. The diff replay is NOT the thing under test — delete every diff row first, so the only
+  // surviving source for the trie is `state:`. 39 of 44 reporting fleet nodes published ZERO here.
+  sm = await open(dir);
+  await sm.db.clear({ gte: 'diff:', lt: 'diff:\xFF' });
+  await sm.db.close();
+  sm = await open(dir);
+  assert.strictEqual(sm.stateTree.state.size, 30, 'keys must come back');
+  assert.notStrictEqual(sm.stateTree.getRoot(), ZERO, 'a tree holding 30 keys must not commit to zeros');
+  assert.strictEqual(sm.stateTree.getRoot(), before, 'and it must be the SAME root it had before the restart');
+  await sm.db.close();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+test('re-submitting the same anchor under fresh tx ids writes ONE durable row, not one per submission', async () => {
+  const dir = `/tmp/xvsm-rehydrate-${process.pid}-b`;
+  rmSync(dir, { recursive: true, force: true });
+  const sm = await open(dir);
+  const set = anchors(50);
+  for (let pass = 0; pass < 4; pass++) for (const a of set) await sm._handleLedgerBlock(blockFor(a, pass));
+  assert.strictEqual(await countDiffRows(sm), 50, '200 submissions of 50 anchors is 50 state changes');
+  assert.strictEqual(sm.getStatistics().totalTransactions, 50, 'and the published count must agree with disk');
+  await sm.db.close();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+test('the canonical set a node adopts SURVIVES a restart', async () => {
+  const dir = `/tmp/xvsm-rehydrate-${process.pid}-c`;
+  rmSync(dir, { recursive: true, force: true });
+  let sm = await open(dir);
+  const local = anchors(50);
+  for (const a of local) await sm._handleLedgerBlock(blockFor(a, 'local'));
+  const canonical = local.slice(0, 10);
+  const reb = await sm.rebuildFromCanonical(canonical);
+  assert.strictEqual(reb.applied, 10);
+  const canonicalRoot = reb.state_root;
+  await sm.db.close();
+
+  sm = await open(dir);
+  assert.strictEqual(sm.stateTree.state.size, 10, 'the 40 non-canonical keys must not come back');
+  assert.strictEqual(sm.stateTree.getRoot(), canonicalRoot, 'the node must still be on the canonical root');
+  assert.strictEqual(sm.getStatistics().totalTransactions, 10);
+  await sm.db.close();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+test('legacy rows keyed by block id are re-keyed to content identity on the next boot', async () => {
+  const dir = `/tmp/xvsm-rehydrate-${process.pid}-d`;
+  rmSync(dir, { recursive: true, force: true });
+  let sm = await open(dir);
+  // Write rows the way every node on the fleet already has them: one per submission, keyed by block id.
+  const set = anchors(40);
+  let written = 0;
+  for (let pass = 0; pass < 3; pass++) for (const a of set) {
+    const key = `anchor:${a.event}:${a.hash}`;
+    await sm.db.put(`diff:blk-${pass}-${a.hash}`, JSON.stringify({
+      txId: `blk-${pass}-${a.hash}`, timestamp: 1000 + pass, changes: { [key]: { ts: a.ts } },
+    }));
+    written++;
+  }
+  assert.strictEqual(await countDiffRows(sm), written);
+  await sm.db.close();
+
+  sm = await open(dir);
+  assert.strictEqual(await countDiffRows(sm), 40, `${written} legacy rows collapse to 40 distinct changes`);
+  assert.strictEqual(sm.getStatistics().totalTransactions, 40);
+  await sm.db.close();
+  rmSync(dir, { recursive: true, force: true });
+});

package/src/state-diff.js

@@ -5,6 +5,17 @@ export class StateDiff {
     this.changes = changes; // key -> new value
   }
 
+  // THE IDENTITY OF A STATE CHANGE IS THE KEY IT WRITES, not the id of whatever block happened to carry it.
+  // A node mints a fresh tx id every time it re-submits an anchor, so `diff:<txId>` made one durable row per
+  // SUBMISSION: measured on a live node 2026-09-15, 92,505 rows over 50,497 distinct anchors — 42,008 of them
+  // redundant, still growing by ~34 every seven hours, and every one of them replayed into the tree on boot.
+  // Keying by the change itself makes a re-submission an upsert, which is what it always was semantically.
+  // Multi-key changes (explicit `state_diff` txs) have no single content key, so they keep the tx id.
+  identity() {
+    const keys = Object.keys(this.changes || {});
+    return keys.length === 1 ? keys[0] : this.txId;
+  }
+
   apply(state) {
     const newState = { ...state };
     for (const [key, value] of Object.entries(this.changes)) {

package/src/state-machine.js

@@ -32,8 +32,10 @@ export class StateMachine extends EventEmitter {
     // Integration: xclt for state commitments from ledger
     this.xclt = options.xclt || null;
     
-    // Initialize database
-    this._initDb().catch(() => {});
+    // Initialize database. KEEP THE PROMISE: `_dbOpen` flips true the moment Level opens, long before the
+    // diff sweep and the tree rehydration have finished, so anything that polls `_dbOpen` and then reads a
+    // count is reading a store mid-rebuild. `ready()` is the only honest join point.
+    this._ready = this._initDb().catch(() => {});
     
     // Listen to ledger events if available
     if (this.xclt) {
@@ -51,10 +53,16 @@ export class StateMachine extends EventEmitter {
     }
   }
   
+  // Resolves once the store is open, the tree rehydrated from `state:`, and the diff rows swept and loaded.
+  ready() { return this._ready || Promise.resolve(); }
+
   async _initDb() {
     try {
       await this.db.open();
       this._dbOpen = true;
+      // The tree rehydrates itself from the `state:` keyspace; wait for it before deciding whether the diff
+      // rows still need to be replayed, or an unfinished load reads as an empty tree.
+      await this.stateTree.ready();
       await this._loadDiffs();
       await this._loadTransactionLog();
     } catch (error) {
@@ -72,24 +80,46 @@ export class StateMachine extends EventEmitter {
       // Replay is safe to do in iterator order because the tree is a key->value map: the root is a function
       // of the final key set, not of application order (proven in verkle-integration.test.mjs, "same set in
       // any order yields the SAME root"). Later diffs for the same key legitimately overwrite earlier ones.
+      // RE-KEY SWEEP. Rows written before content identity existed are keyed `diff:<block.id>`, so the same
+      // anchor re-submitted under a fresh tx id left one row per submission and every one of them came back
+      // here on boot. Rewriting each row under `diff:<identity>` collapses them: the identity-keyed row is
+      // PUT first and the old key deleted only after, so nothing is lost if this is interrupted. Measured on
+      // a live store 2026-09-15: 92,505 rows over 50,497 distinct anchors, 42,008 of them redundant.
       const loaded = [];
+      let rekeyed = 0;
       for await (const [key, value] of this.db.iterator({ gt: 'diff:', lt: 'diff:\xFF' })) {
         const diffData = JSON.parse(value.toString());
         const diff = new StateDiff(diffData.txId, diffData.changes);
         diff.timestamp = diffData.timestamp;
-        this._recordDiff(diff);
-        loaded.push(diff);
+        const want = this._diffKey(diff);
+        const have = key.toString();
+        if (have !== want) {
+          try { await this.db.put(want, value); await this.db.del(have); rekeyed++; } catch { /* leave it */ }
+        }
+        if (this._recordDiff(diff)) loaded.push(diff);
       }
-      loaded.sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0) || String(a.txId).localeCompare(String(b.txId)));
+      if (rekeyed) console.log(`[XVSM] diff rows re-keyed to content identity: ${rekeyed}, distinct now ${this.diffs.length}`);
+
+      // ⛔ REPLAY ONLY AS RECOVERY. This loop replayed every diff into the tree unconditionally, which is how
+      // it papered over the real defect — VerkleStateTree._loadState restored the key map but never rebuilt
+      // the trie, so the root read 64 zeros and the replay was the only thing putting keys back through
+      // insert(). With the trie rebuilt on open, an unconditional replay is actively harmful: it reinstates
+      // every key that rebuildFromCanonical deliberately dropped, so the canonical root a node adopts SURVIVES
+      // until its next restart and no further. MEASURED 2026-09-15: rebuild to root 02eaf3e76c over 10
+      // canonical anchors, restart, root 4bc7dd2813 over 50 keys — the node left the canonical set by booting.
+      // Replay stays for the one case it is still needed: a store whose `state:` keyspace is genuinely empty.
       let replayed = 0;
-      for (const diff of loaded) {
-        for (const [k, v] of Object.entries(diff.changes || {})) {
-          if (v === null) { await this.stateTree.delete?.(k); continue; }
-          await this.stateTree.insert(k, v);
-          replayed++;
+      if (this.stateTree.state.size === 0 && loaded.length) {
+        loaded.sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0) || String(a.txId).localeCompare(String(b.txId)));
+        for (const diff of loaded) {
+          for (const [k, v] of Object.entries(diff.changes || {})) {
+            if (v === null) { await this.stateTree.delete?.(k); continue; }
+            await this.stateTree.insert(k, v);
+            replayed++;
+          }
         }
       }
-      if (replayed) console.log(`[XVSM] verkle tree rehydrated: ${replayed} change(s) from ${loaded.length} diff(s), root=${this.stateTree.getRoot().slice(0, 16)}…`);
+      if (replayed) console.log(`[XVSM] verkle tree recovered from diffs (state: keyspace was empty): ${replayed} change(s) from ${loaded.length} diff(s), root=${this.stateTree.getRoot().slice(0, 16)}…`);
     } catch (error) {
       // Ignore load errors
     }
@@ -112,7 +142,7 @@ export class StateMachine extends EventEmitter {
     if (!this._dbOpen) return;
     
     try {
-      await this.db.put(`diff:${diff.txId}`, JSON.stringify({
+      await this.db.put(this._diffKey(diff), JSON.stringify({
         txId: diff.txId,
         changes: diff.changes,
         timestamp: diff.timestamp
@@ -153,13 +183,22 @@ export class StateMachine extends EventEmitter {
   //
   // REPLACE, don't skip: re-applying a txId with different changes is a legitimate later state for that key
   // and the disk already resolves it that way. Skipping would leave the array disagreeing with the tree.
+  //
+  // INDEXED BY CONTENT IDENTITY, NOT BY TX ID, so the in-memory set matches the durable one key-for-key. Two
+  // submissions of the same anchor carry different tx ids and are the SAME state change; keying either side
+  // by the tx id makes them two of everything. See StateDiff.identity().
   _recordDiff(diff) {
-    const at = this._diffIndex.get(diff.txId);
-    if (at === undefined) { this._diffIndex.set(diff.txId, this.diffs.length); this.diffs.push(diff); return true; }
+    const id = diff.identity();
+    const at = this._diffIndex.get(id);
+    if (at === undefined) { this._diffIndex.set(id, this.diffs.length); this.diffs.push(diff); return true; }
     this.diffs[at] = diff;
     return false;
   }
 
+  // The one place a diff's durable key is spelled. Every writer goes through it, or the re-key sweep in
+  // _loadDiffs is fighting a writer that still uses the block id.
+  _diffKey(diff) { return `diff:${diff.identity()}`; }
+
   _stateChangesFor(block) {
     const tx = block?.tx;
     if (!tx || typeof tx !== 'object') return null;
@@ -208,7 +247,7 @@ export class StateMachine extends EventEmitter {
         // StateDiff.serialize() ALREADY returns a JSON string — wrapping it in JSON.stringify again
         // double-encodes, so _loadDiffs parses back a string instead of an object and `changes` comes out
         // undefined, silently replaying nothing. Store the serialized form directly.
-        try { await this.db.put(`diff:${block.id}`, diff.serialize()); }
+        try { await this.db.put(this._diffKey(diff), diff.serialize()); }
         catch { /* in-memory fallback */ }
       }
     } catch (error) {
@@ -252,7 +291,7 @@ export class StateMachine extends EventEmitter {
         this._recordDiff(diff);
         for (const [key, val] of Object.entries(changes)) await this.stateTree.insert(key, val);
         if (this._dbOpen !== false) {
-          try { await this.db.put(`diff:${block.id}`, diff.serialize()); } catch { /* in-memory fallback */ }
+          try { await this.db.put(this._diffKey(diff), diff.serialize()); } catch { /* in-memory fallback */ }
         }
         out.applied++;
       } catch { out.failed++; }
@@ -273,6 +312,18 @@ export class StateMachine extends EventEmitter {
     await this.stateTree.clear();
     this.diffs = [];
     this._diffIndex.clear();
+    // ⛔ THE DURABLE ROWS COME TOO. Clearing `this.diffs` in memory while 92,505 `diff:` rows stay on disk is
+    // not an authoritative rebuild — it is one that lasts until the next boot reads them back. The tree's
+    // `state:` keyspace is cleared above for exactly this reason; the diff keyspace is the same commitment
+    // written twice, and leaving half of it behind is what made the canonical set un-adoptable across a
+    // restart. The blocks themselves are untouched in the ledger: backfillFromLedger regenerates local
+    // history on demand, and the canonical rows are re-persisted below.
+    if (this._dbOpen) {
+      try {
+        if (typeof this.db.clear === 'function') await this.db.clear({ gte: 'diff:', lt: 'diff:\xFF' });
+        else for await (const [k] of this.db.iterator({ gte: 'diff:', lt: 'diff:\xFF' })) { try { await this.db.del(k); } catch { /* */ } }
+      } catch { /* best-effort */ }
+    }
     // ⛔ RECORD A DIFF FOR EVERY ANCHOR APPLIED, or applied_tx_count IS ZERO BY CONSTRUCTION. This loop wrote
     // straight into the tree and never touched `this.diffs`, which it had just emptied — and
     // getStatistics().totalTransactions is transactionLog.length + diffs.length. So the moment a node runs the
@@ -288,7 +339,9 @@ export class StateMachine extends EventEmitter {
         const key = `anchor:${a.event}:${a.hash}`;
         const value = { ts: a.ts ?? null };
         await this.stateTree.insert(key, value);
-        this._recordDiff(new StateDiff(key, { [key]: value }));
+        const diff = new StateDiff(key, { [key]: value });
+        this._recordDiff(diff);
+        if (this._dbOpen) { try { await this.db.put(this._diffKey(diff), diff.serialize()); } catch { /* in-memory fallback */ } }
         out.applied++;
       } catch { out.skipped++; }
     }

package/src/verkle-tree.js

@@ -16,9 +16,10 @@ export class VerkleStateTree {
     this.db = options.db || null;
     this._dbOpen = false;
     
-    if (this.db) {
-      this._initDb().catch(() => {});
-    }
+    // KEEP THE PROMISE. This was fire-and-forget, so XVSM's own _loadDiffs raced the tree's _loadState and
+    // could not tell "the tree is empty" from "the tree has not finished loading" — the exact distinction the
+    // replay decision below now turns on. `ready()` is the join point.
+    this._ready = this.db ? this._initDb().catch(() => {}) : Promise.resolve();
   }
   
   async _initDb() {
@@ -33,6 +34,9 @@ export class VerkleStateTree {
     }
   }
   
+  // Resolves once the `state:` keyspace has been read and the trie rebuilt from it.
+  ready() { return this._ready || Promise.resolve(); }
+
   async _loadState() {
     if (!this.db || !this._dbOpen) return;
     
@@ -45,6 +49,30 @@ export class VerkleStateTree {
     } catch (error) {
       // Ignore errors during load
     }
+    // ⛔ REBUILD THE TRIE. This loop restored `this.state` — the key->value map — and stopped there, leaving
+    // `this.root` a fresh empty VerkleNode. So a node that had loaded EVERY key off disk answered getRoot()
+    // with 64 zeros: a tree that holds the whole state and commits to nothing. MEASURED 2026-09-15: after a
+    // canonical rebuild of 10 anchors, restart loaded all 10 keys and published root 0000…0000; across the
+    // fleet 39 of 44 reporting nodes published exactly that, including one with 11,691 blocks persisted. The
+    // only boxes showing a real root were showing it by accident, via XVSM's diff replay re-inserting keys
+    // through insert(). The state map is not the commitment — the hashed trie over it is, and it has to be
+    // reconstructed on open or every restart silently un-commits the node's entire state.
+    this._rebuildTrie();
+  }
+
+  // Rebuild the hashed trie from `this.state` in ONE bottom-up pass. Per-key _updateHashPath would re-hash
+  // every ancestor once per key (32 levels x 8KB of child hashes each); this inserts every key first and
+  // hashes each node exactly once, so a 50k-key store rehydrates in a single walk instead of 1.6M of them.
+  _rebuildTrie() {
+    this.root = new VerkleNode();
+    for (const [k, v] of this.state) {
+      this._insertNode(this.root, this._hashKey(k), this._hashValue(v), 0, []);
+    }
+    const hashSubtree = (node) => {
+      for (const child of node.children.values()) hashSubtree(child);
+      this._updateHash(node);
+    };
+    hashSubtree(this.root);
   }
   
   async _saveState(key, value) {