@typeberry/lib 0.9.0 → 0.10.0

package/packages/jam/database-fjall/hybrid-states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;GAUG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAsBxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;WAxBZ,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,GACV,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAIX,mDAAmD;YACrC,WAAW;IAgBzB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}
+{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAa,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAgB,MAAM,WAAW,CAAC;AAI3F;;;;;;;;;GASG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,SAAS;IAH5B,OAAO;IAMP,4EAA4E;WAC/D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAc9F,mEAAmE;IAC7D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,wDAAwD;IACxD,WAAW,IAAI,MAAM,GAAG,IAAI;IAI5B,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0CxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,WAAW;WA7CjB,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,cAAc,GACf,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC;;;;;;;OAOG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,GAAG,sBAAsB;IAI1G,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,OAAO;IAWD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAIhC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAcX,KAAK;IAWX,qEAAqE;YACvD,WAAW;IAoBzB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-fjall/hybrid-states.js

@@ -1,40 +1,114 @@
 import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
 import { Logger } from "#@typeberry/logger";
 import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
 import { leafComparator } from "#@typeberry/trie";
 import { OK, Result } from "#@typeberry/utils";
 import { FjallRoot, toUint8Array } from "./root.js";
 const logger = Logger.new(import.meta.filename, "db");
+/**
+ * One open fjall keyspace together with its content-addressed `values`
+ * partition.
+ *
+ * Opening the keyspace is the slow part, so the fuzz target opens one session
+ * per run and reuses it for every reset (see `HybridSerializedStates.fromSession`).
+ * The values partition is immutable - the key is the hash of the value - so it
+ * is fine that values pile up across resets, the unreferenced ones just sit
+ * there unused.
+ */
+export class FjallValuesSession {
+    root;
+    values;
+    constructor(root, 
+    /** Shared content-addressed values partition, reused across resets. */
+    values) {
+        this.root = root;
+        this.values = values;
+    }
+    /** Open (or create) the keyspace at `dbPath` and its `values` partition. */
+    static async open(dbPath, options = {}) {
+        if (options.readOnly === true) {
+            throw new Error("FjallValuesSession requires a writable keyspace.");
+        }
+        const root = await FjallRoot.open(dbPath, options);
+        try {
+            const values = await root.writablePartition("values");
+            return new FjallValuesSession(root, values);
+        }
+        catch (e) {
+            await root.close();
+            throw e;
+        }
+    }
+    /** Flush the journal to disk (a no-op for ephemeral keyspaces). */
+    async persist() {
+        await this.root.persist();
+    }
+    /** Size of the keyspace directory on disk, in bytes. */
+    sizeInBytes() {
+        return this.root.sizeInBytes();
+    }
+    /** Release the keyspace handle (skips the sync-all fsync when ephemeral). */
+    async close() {
+        await this.root.close();
+    }
+}
 /**
  * Hybrid serialized-states db (fjall variant).
  *
- * States (leafs) are kept in-memory, but large values are persisted to fjall.
- * Reads go straight to fjall, which keeps its own (bounded) block cache.
- * Designed for long fuzzing, used with pruning to keep heap usage bounded.
+ * States (leafs) are kept in memory, only the large values go to fjall on disk.
+ * Reads hit fjall directly, which keeps its own bounded block cache. Meant for
+ * long fuzzing, used together with pruning so the heap stays bounded.
+ *
+ * Construction is async, and value writes are flushed explicitly, because fjall
+ * has no transaction primitive.
  *
- * Behaviourally identical to the LMDB hybrid db; differences are mechanical:
- * construction is async, and value writes are ordered + flushed explicitly
- * because fjall has no transaction primitive.
+ * Values that no longer belong to any surviving state are removed from fjall,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous run
+ * are never collected. An instance backed by a shared session (fuzz reset
+ * reuse) only ever prunes values it inserted itself, since its refcounts start
+ * empty - values left behind by earlier resets stay untouched.
  */
 export class HybridSerializedStates {
     spec;
     blake2b;
-    root;
-    values;
-    static async new({ spec, blake2b, dbPath, ephemeral, }) {
-        const root = await FjallRoot.open(dbPath, { ephemeral });
-        const values = await root.partition("values");
-        return new HybridSerializedStates(spec, blake2b, root, values);
+    session;
+    ownsSession;
+    static async new({ spec, blake2b, dbPath, ephemeral, cacheSizeBytes, }) {
+        const session = await FjallValuesSession.open(dbPath, { ephemeral, cacheSizeBytes });
+        // This instance owns the session it just opened, so its `close()` closes it.
+        return new HybridSerializedStates(spec, blake2b, session, true);
+    }
+    /**
+     * Wrap an already-open `FjallValuesSession` and reuse its keyspace.
+     *
+     * The new instance starts with its own empty in-memory leaf sets but shares
+     * the values partition on disk. Its `close()` does not close the session, the
+     * session owner closes it once. The fuzz target uses this to keep one keyspace
+     * across resets and only rebuild the in-memory state for each vector.
+     */
+    static fromSession(spec, blake2b, session) {
+        return new HybridSerializedStates(spec, blake2b, session, false);
     }
     inMemStates = HashDictionary.new();
     // A single shared values accessor reused by every `LeafDb` we hand out.
     valuesDb;
-    constructor(spec, blake2b, root, values) {
+    /** Shared content-addressed values partition (owned by `session`). */
+    values;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
+    constructor(spec, blake2b, session, 
+    /** Whether `close()` should close the underlying session. */
+    ownsSession) {
         this.spec = spec;
         this.blake2b = blake2b;
-        this.root = root;
-        this.values = values;
+        this.session = session;
+        this.ownsSession = ownsSession;
+        this.values = session.values;
         this.valuesDb = { get: (key) => this.readValue(key) };
     }
     async insertInitialState(headerHash, entries) {
@@ -44,13 +118,14 @@ export class HybridSerializedStates {
             return res;
         }
         this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async updateAndSetState(header, state, update) {
         const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
         // Clone the leaf set before mutating: the previous state keeps using its own.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
         const res = await this.writeValues(values);
         if (res.isError) {
             // Leave the caller's state untouched: its new leaves would reference
@@ -61,6 +136,7 @@ export class HybridSerializedStates {
         // values are durably written.
         state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
         this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -74,27 +150,56 @@ export class HybridSerializedStates {
         const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
-        // We only remove the state from memory - values are not pruned at all,
-        // but since they are stored on disk we should be safe.
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.inMemStates.delete(header);
     }
     diskSizeInBytes() {
-        return this.root.sizeInBytes();
+        return this.session.sizeInBytes();
+    }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value. No explicit
+        // persist - removals are flushed together with the next value write.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => Promise.all(update.removeValues.map((v) => this.values.remove(v.raw))))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
     }
     async close() {
-        await this.root.close();
+        // Finish any queued value removals before tearing down (or releasing) the
+        // session - they operate on the shared `values` partition.
+        await this.pendingCleanup;
+        // Instances backed by a shared session (fuzz reset reuse) keep the keyspace
+        // open for the next reset. The session owner closes it once.
+        if (this.ownsSession) {
+            await this.session.close();
+        }
     }
-    /** Write new large values to fjall, then flush. */
+    /** Write new large values to fjall in a single batch, then flush. */
     async writeValues(values) {
         if (values.length === 0) {
             return Result.ok(OK);
         }
         try {
-            for (const [hash, val] of values) {
-                await this.values.insert(hash.raw, val.raw);
-            }
-            await this.root.persist();
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
+            const entries = values.map(([hash, val]) => ({ key: hash.raw, value: val.raw }));
+            await this.values.insertBatch(entries);
+            await this.session.persist();
         }
         catch (e) {
             logger.error `${e}`;

package/packages/jam/database-fjall/hybrid-states.test.js

@@ -1,13 +1,15 @@
 import assert from "node:assert";
 import * as fs from "node:fs";
 import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { tryAsServiceId } from "#@typeberry/block";
 import { Bytes, BytesBlob } from "#@typeberry/bytes";
 import { tinyChainSpec } from "#@typeberry/config";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
-import { InMemoryState } from "#@typeberry/state";
+import { InMemoryState, StorageItem, UpdateStorage, } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
 import { StateEntries } from "#@typeberry/state-merkleization";
-import { deepEqual, OK, Result } from "#@typeberry/utils";
-import { HybridSerializedStates } from "./hybrid-states.js";
+import { asOpaqueType, deepEqual, OK, Result } from "#@typeberry/utils";
+import { FjallValuesSession, HybridSerializedStates } from "./hybrid-states.js";
 let blake2b;
 before(async () => {
     blake2b = await Blake2b.createHasher();
@@ -66,6 +68,36 @@ describe("Fjall hybrid serialized states", () => {
             await states.close();
         }
     });
+    it("shares an open values session across resets without closing it", async () => {
+        // The fuzz reset path opens the values keyspace once per session and reuses
+        // it: each "reset" builds a fresh states instance sharing that session, and
+        // closing a session-backed states must NOT close the shared keyspace.
+        const session = await FjallValuesSession.open(dbPath);
+        try {
+            const big = BytesBlob.blobFromString("z".repeat(100));
+            const key = Bytes.fill(HASH_SIZE, 7).asOpaque();
+            const entries = StateEntries.fromEntriesUnsafe([[key, big]]);
+            // First "reset": write values through a states instance, then close it.
+            const first = HybridSerializedStates.fromSession(spec, blake2b, session);
+            const res = await first.insertInitialState(headerHash, entries);
+            deepEqual(res, Result.ok(OK));
+            await first.close();
+            // Second "reset": a fresh states sharing the same session. Its in-memory
+            // leaf set is independent (empty until it inserts)...
+            const second = HybridSerializedStates.fromSession(spec, blake2b, session);
+            assert.strictEqual(second.getState(headerHash), null);
+            // ...but the on-disk values store is the same one, still open and usable
+            // (a closed keyspace would throw here).
+            await second.insertInitialState(headerHash, entries);
+            const state = second.getState(headerHash);
+            assert.ok(state !== null);
+            assert.strictEqual(`${state.backend.get(key)}`, `${big}`);
+            await second.close();
+        }
+        finally {
+            await session.close();
+        }
+    });
     it("drops the leaf set on markUnused while values stay on disk", async () => {
         const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
         try {
@@ -81,3 +113,93 @@ describe("Fjall hybrid serialized states", () => {
         }
     });
 });
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+const storageKey = asOpaqueType(BytesBlob.blobFromString("test-key"));
+/** A state update writing a single large (non-embedded) value under `storageKey`. */
+function storageUpdate(value) {
+    const item = StorageItem.create({ key: storageKey, value: BytesBlob.blobFromString(value) });
+    return {
+        storage: new Map([[tryAsServiceId(0), [UpdateStorage.set({ storage: item })]]]),
+    };
+}
+// > 32 bytes => stored in the values db (not embedded in the leaf).
+const BIG_1 = "a".repeat(100);
+const BIG_2 = "b".repeat(100);
+/** `true` if every value referenced by the state can still be resolved. */
+function canReadFully(state) {
+    if (state === null) {
+        return false;
+    }
+    try {
+        state.backend.intoStateEntries();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Value removals are queued, so poll for the expected outcome. */
+async function eventually(check, what, timeoutMs = 5_000) {
+    const start = Date.now();
+    while (!check()) {
+        if (Date.now() - start > timeoutMs) {
+            assert.fail(`Timed out waiting for: ${what}`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+}
+describe("Fjall hybrid serialized states value refcounting", () => {
+    const spec = tinyChainSpec;
+    let dbPath = "";
+    beforeEach(() => {
+        dbPath = createTempDir();
+    });
+    afterEach(() => {
+        fs.rmSync(dbPath, { recursive: true });
+    });
+    it("removes a replaced value from disk once the replacement finalizes", async () => {
+        const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a handle to the post-1 state, surviving the pruning below
+            const stale1 = states.getState(hh(1));
+            const s1 = states.getState(hh(1));
+            assert.ok(s1 !== null);
+            await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
+            states.commitFinalized([hh(1)]);
+            assert.ok(canReadFully(stale1), "still referenced by the finalized tip");
+            states.commitFinalized([hh(2)]);
+            await eventually(() => !canReadFully(stale1), "replaced value removed from fjall");
+            assert.ok(canReadFully(states.getState(hh(2))), "the new finalized tip stays fully readable");
+        }
+        finally {
+            await states.close();
+        }
+    });
+    it("collects values of a pruned dead fork and keeps the surviving chain intact", async () => {
+        const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a dead fork on top of genesis, inserting a different value
+            const fork = states.getState(hh(0));
+            assert.ok(fork !== null);
+            await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_2));
+            const staleFork = states.getState(hh(0xaa));
+            states.markUnused(hh(0xaa));
+            assert.strictEqual(states.getState(hh(0xaa)), null);
+            await eventually(() => !canReadFully(staleFork), "fork-only value removed from fjall");
+            assert.ok(canReadFully(states.getState(hh(1))), "surviving chain is unaffected");
+        }
+        finally {
+            await states.close();
+        }
+    });
+});

package/packages/jam/database-fjall/index.d.ts

@@ -1,3 +1,5 @@
+export * from "./blocks.js";
 export * from "./hybrid-states.js";
 export * from "./root.js";
+export * from "./states.js";
 //# sourceMappingURL=index.d.ts.map

package/packages/jam/database-fjall/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}

package/packages/jam/database-fjall/index.js

@@ -1,2 +1,4 @@
+export * from "./blocks.js";
 export * from "./hybrid-states.js";
 export * from "./root.js";
+export * from "./states.js";

package/packages/jam/database-fjall/root.d.ts

@@ -1,25 +1,33 @@
-import { type Partition } from "@fjall-js/fjall";
-export type { Partition };
+import { type Partition, type ReadonlyPartition } from "@fjall-js/fjall";
+export type { Partition, ReadonlyPartition };
+export type FjallPartition = Partition | ReadonlyPartition;
 /**
  * Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
  */
 export declare function toUint8Array(value: Buffer | null): Uint8Array | null;
 export type FjallRootOptions = {
+    /** Open a read-only wrapper surface over the shared keyspace. */
+    readOnly?: boolean;
     /**
-     * When set, durability flushes (`persist`) are skipped entirely.
+     * When set, we skip explicit durability flushes (`persist`).
      *
-     * Only safe for throwaway databases (e.g. the fuzz target, which wipes on
-     * every reset). Mirrors LMDB's `noSync`.
+     * Only safe for throwaway databases, like the fuzz target that wipes on every
+     * reset.
      */
     ephemeral?: boolean;
+    /**
+     * Cache size in bytes, shared by all partitions of the keyspace. fjall reads
+     * through this cache, so it bounds how much we keep in memory. When not set,
+     * typeberry uses a conservative default.
+     */
+    cacheSizeBytes?: number;
 };
 /**
- * A thin abstraction over the fjall keyspace.
+ * Thin wrapper over the fjall keyspace.
  *
- * Unlike LMDB (a memory-mapped B-tree), fjall is an LSM-tree that reads/writes
- * through regular file I/O, so its working set is bounded by an explicit block
- * cache rather than the whole mmap. LMDB has been causing oom issues because
- * of that.
+ * fjall is an LSM-tree: it reads and writes through normal file i/o and keeps
+ * only a bounded block cache in memory, so the resident set stays bounded even
+ * when the store on disk is big.
  */
 export declare class FjallRoot {
     private readonly keyspace;
@@ -28,9 +36,13 @@ export declare class FjallRoot {
     private readonly options;
     private constructor();
     /** Open (or create) a fjall keyspace at the given path. */
-    static open(dbPath: string, options: FjallRootOptions): Promise<FjallRoot>;
+    static open(dbPath: string, options?: FjallRootOptions): Promise<FjallRoot>;
+    /** Whether this root was opened through fjall's read-only surface. */
+    get readOnly(): boolean;
     /** Open (or create) a partition under this keyspace. */
-    partition(name: string): Promise<Partition>;
+    partition(name: string): Promise<FjallPartition>;
+    /** Open a writable partition, failing early when this root is read-only. */
+    writablePartition(name: string): Promise<Partition>;
     /**
      * Flush the journal to disk so prior writes survive a crash.
      *
@@ -39,14 +51,14 @@ export declare class FjallRoot {
      */
     persist(): Promise<void>;
     /**
-     * Apparent on-disk size of the keyspace directory, in bytes.
+     * Size of the keyspace directory on disk, in bytes.
      *
-     * Returns `null` if the directory cannot be walked (e.g. not yet created).
-     * Unlike LMDB's single `data.mdb`, a fjall keyspace is a directory of
-     * partition and journal files, so we sum it recursively.
+     * Returns `null` when the directory cannot be walked (e.g. not created yet).
+     * A fjall keyspace is a directory of partition and journal files, so we sum
+     * them recursively.
      */
     sizeInBytes(): number | null;
-    /** Persist with `sync-all` and release the keyspace handle. */
+    /** Release this keyspace handle. Call persist() first when durability is needed. */
     close(): Promise<void>;
 }
 //# sourceMappingURL=root.d.ts.map

package/packages/jam/database-fjall/root.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEtE,YAAY,EAAE,SAAS,EAAE,CAAC;AAE1B;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAKpE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,SAAS;IAElB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO;IAOP,2DAA2D;WAC9C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC;IAKhF,wDAAwD;IAClD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAIjD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;;;;;OAMG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
+{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,SAAS,EAEd,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAIzB,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC;AAC7C,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;AAE3D;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAKpE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,SAAS;IAElB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO;IAOP,2DAA2D;WAC9C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,SAAS,CAAC;IAWrF,sEAAsE;IACtE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,wDAAwD;IAClD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAItD,4EAA4E;IACtE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAOzD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAU9B;;;;;;OAMG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,oFAAoF;IAC9E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}

package/packages/jam/database-fjall/root.js

@@ -1,6 +1,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { open } from "@fjall-js/fjall";
+import { open, openReadonly, } from "@fjall-js/fjall";
+const DEFAULT_CACHE_SIZE_BYTES = 64 * 1024 * 1024;
 /**
  * Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
  */
@@ -11,12 +12,11 @@ export function toUint8Array(value) {
     return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
 }
 /**
- * A thin abstraction over the fjall keyspace.
+ * Thin wrapper over the fjall keyspace.
  *
- * Unlike LMDB (a memory-mapped B-tree), fjall is an LSM-tree that reads/writes
- * through regular file I/O, so its working set is bounded by an explicit block
- * cache rather than the whole mmap. LMDB has been causing oom issues because
- * of that.
+ * fjall is an LSM-tree: it reads and writes through normal file i/o and keeps
+ * only a bounded block cache in memory, so the resident set stays bounded even
+ * when the store on disk is big.
  */
 export class FjallRoot {
     keyspace;
@@ -30,14 +30,31 @@ export class FjallRoot {
         this.options = options;
     }
     /** Open (or create) a fjall keyspace at the given path. */
-    static async open(dbPath, options) {
-        const keyspace = await open(dbPath);
+    static async open(dbPath, options = {}) {
+        // fjall-js 0.3 shares one engine per path. Readers must use the read-only
+        // wrapper surface; durability is driven explicitly through persist().
+        const config = {
+            path: dbPath,
+            cacheSizeBytes: options.cacheSizeBytes ?? DEFAULT_CACHE_SIZE_BYTES,
+        };
+        const keyspace = options.readOnly === true ? await openReadonly(config) : await open(config);
         return new FjallRoot(keyspace, dbPath, options);
     }
+    /** Whether this root was opened through fjall's read-only surface. */
+    get readOnly() {
+        return this.options.readOnly === true;
+    }
     /** Open (or create) a partition under this keyspace. */
     async partition(name) {
         return this.keyspace.partition(name);
     }
+    /** Open a writable partition, failing early when this root is read-only. */
+    async writablePartition(name) {
+        if (this.readOnly) {
+            throw new Error(`Cannot open writable fjall partition '${name}' from a read-only keyspace.`);
+        }
+        return (await this.keyspace.partition(name));
+    }
     /**
      * Flush the journal to disk so prior writes survive a crash.
      *
@@ -48,14 +65,17 @@ export class FjallRoot {
         if (this.options.ephemeral === true) {
             return;
         }
+        if (this.readOnly) {
+            throw new Error("Cannot persist a read-only fjall keyspace.");
+        }
         await this.keyspace.persist();
     }
     /**
-     * Apparent on-disk size of the keyspace directory, in bytes.
+     * Size of the keyspace directory on disk, in bytes.
      *
-     * Returns `null` if the directory cannot be walked (e.g. not yet created).
-     * Unlike LMDB's single `data.mdb`, a fjall keyspace is a directory of
-     * partition and journal files, so we sum it recursively.
+     * Returns `null` when the directory cannot be walked (e.g. not created yet).
+     * A fjall keyspace is a directory of partition and journal files, so we sum
+     * them recursively.
      */
     sizeInBytes() {
         try {
@@ -65,7 +85,7 @@ export class FjallRoot {
             return null;
         }
     }
-    /** Persist with `sync-all` and release the keyspace handle. */
+    /** Release this keyspace handle. Call persist() first when durability is needed. */
     async close() {
         await this.keyspace.close();
     }

package/packages/jam/database-fjall/states.d.ts

@@ -0,0 +1,31 @@
+import type { HeaderHash, StateRootHash } from "#@typeberry/block";
+import type { ChainSpec } from "#@typeberry/config";
+import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
+import type { Blake2b } from "#@typeberry/hash";
+import type { ServicesUpdate, State } from "#@typeberry/state";
+import type { StateEntries } from "#@typeberry/state-merkleization";
+import { SerializedState } from "#@typeberry/state-merkleization";
+import { OK, Result } from "#@typeberry/utils";
+import { type FjallRoot } from "./root.js";
+/** fjall-backed full on-disk serialized state storage. */
+export declare class FjallStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
+    private readonly spec;
+    private readonly blake2b;
+    private readonly root;
+    private readonly states;
+    private readonly values;
+    static open(spec: ChainSpec, blake2b: Blake2b, root: FjallRoot): Promise<FjallStates>;
+    private readonly valuesDb;
+    private constructor();
+    insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
+    updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
+    getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
+    getState(headerHash: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(_headers: HeaderHash[]): void;
+    markUnused(headerHash: HeaderHash): void;
+    diskSizeInBytes(): number | null;
+    close(): Promise<void>;
+    private updateAndCommit;
+    private readValue;
+}
+//# sourceMappingURL=states.d.ts.map

package/packages/jam/database-fjall/states.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAE/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAI9F,0DAA0D;AAC1D,qBAAa,WAAY,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS7F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;WAZZ,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAK3F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,OAAO;IAUD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAQxG,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAUlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAahE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAK7C,UAAU,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMxC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;YAEG,eAAe;IAyB7B,OAAO,CAAC,SAAS;CAOlB"}