@typeberry/lib 0.9.0 → 0.10.0-3e244fa

package/packages/jam/database-fjall/states.js

@@ -0,0 +1,99 @@
+import { BytesBlob } from "#@typeberry/bytes";
+import { SortedSet } from "#@typeberry/collections";
+import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { Logger } from "#@typeberry/logger";
+import { SerializedState, StateEntryUpdateAction, serializeStateUpdate } from "#@typeberry/state-merkleization";
+import { leafComparator } from "#@typeberry/trie";
+import { OK, Result, resultToString } from "#@typeberry/utils";
+import { toUint8Array } from "./root.js";
+const logger = Logger.new(import.meta.filename, "db");
+/** fjall-backed full on-disk serialized state storage. */
+export class FjallStates {
+    spec;
+    blake2b;
+    root;
+    states;
+    values;
+    static async open(spec, blake2b, root) {
+        const [states, values] = await Promise.all([root.partition("states"), root.partition("values")]);
+        return new FjallStates(spec, blake2b, root, states, values);
+    }
+    valuesDb;
+    constructor(spec, blake2b, root, states, values) {
+        this.spec = spec;
+        this.blake2b = blake2b;
+        this.root = root;
+        this.states = states;
+        this.values = values;
+        this.valuesDb = { get: (key) => this.readValue(key) };
+    }
+    async insertInitialState(headerHash, entries) {
+        return await this.updateAndCommit(headerHash, SortedSet.fromArray(leafComparator, []), Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
+    }
+    async updateAndSetState(headerHash, state, update) {
+        const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
+        const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
+        const res = await this.updateAndCommit(headerHash, newLeafs, updatedValues);
+        if (res.isOk) {
+            state.updateBackend(LeafDb.fromLeaves(newLeafs, this.valuesDb));
+        }
+        return res;
+    }
+    async getStateRoot(state) {
+        return state.backend.getStateRoot(this.blake2b);
+    }
+    getState(headerHash) {
+        const leafNodes = toUint8Array(this.states.get(headerHash.raw));
+        if (leafNodes === null) {
+            return null;
+        }
+        const leafDbResult = LeafDb.fromLeavesBlob(BytesBlob.blobFrom(leafNodes), this.valuesDb);
+        if (leafDbResult.isError) {
+            throw new Error(`Inconsistent DB. Invalid leaf nodes for ${headerHash}: ${resultToString(leafDbResult)}`);
+        }
+        return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
+    }
+    commitFinalized(_headers) {
+        // Values are never pruned here. This db survives restarts, so refcounting
+        // would need counts persisted (and crash-consistent) alongside the values.
+    }
+    markUnused(headerHash) {
+        void writable(this.states, this.root)
+            .remove(headerHash.raw)
+            .catch((e) => logger.warn `Failed to prune state ${headerHash}: ${e}`);
+    }
+    diskSizeInBytes() {
+        return this.root.sizeInBytes();
+    }
+    async close() { }
+    async updateAndCommit(headerHash, leafs, data) {
+        const { values } = updateLeafs(leafs, this.blake2b, data);
+        const stateLeafs = BytesBlob.blobFromParts(leafs.array.map((x) => x.node.raw));
+        try {
+            // Preserve dependency order: values first, then leaves that may reference them.
+            if (values.length > 0) {
+                await writable(this.values, this.root).insertBatch(values.map(([hash, val]) => ({ key: hash.raw, value: val.raw })));
+            }
+            await writable(this.states, this.root).insert(headerHash.raw, stateLeafs.raw);
+            await this.root.persist();
+        }
+        catch (e) {
+            logger.error `${e}`;
+            return Result.error(StateUpdateError.Commit, () => `Failed to commit state update: ${e}`);
+        }
+        return Result.ok(OK);
+    }
+    readValue(key) {
+        const val = toUint8Array(this.values.get(key.raw));
+        if (val === null) {
+            throw new Error(`Missing required value: ${key} in the DB`);
+        }
+        return val;
+    }
+}
+function writable(partition, root) {
+    if (root.readOnly) {
+        throw new Error("Cannot write through a read-only fjall partition.");
+    }
+    return partition;
+}

package/packages/jam/database-fjall/states.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=states.test.d.ts.map

package/packages/jam/database-fjall/states.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.test.ts"],"names":[],"mappings":""}

package/packages/jam/database-fjall/states.test.js

@@ -0,0 +1,150 @@
+import assert from "node:assert";
+import * as fs from "node:fs";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { tinyChainSpec } from "#@typeberry/config";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
+import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
+import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
+import { StateEntries } from "#@typeberry/state-merkleization";
+import { deepEqual, OK, Result } from "#@typeberry/utils";
+import { FjallRoot } from "./root.js";
+import { FjallStates } from "./states.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+describe("Fjall states database", () => {
+    const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
+    const spec = tinyChainSpec;
+    let tmpDir = "";
+    beforeEach(() => {
+        tmpDir = fs.mkdtempSync("typeberry-fjall-states-");
+    });
+    afterEach(() => {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+    it("imports and reads an empty state", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const emptyState = InMemoryState.empty(spec);
+            const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
+            const emptyRoot = serialized.getRootHash(blake2b);
+            deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
+            const newState = states.getState(headerHash);
+            assert.ok(newState !== null);
+            assert.strictEqual(`${await states.getStateRoot(newState)}`, `${emptyRoot}`);
+            deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+    it("reads large values back from disk", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const big1 = BytesBlob.blobFromString("x".repeat(100));
+            const big2 = BytesBlob.blobFromString("y".repeat(100));
+            const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
+            const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
+            const entries = StateEntries.fromEntriesUnsafe([
+                [key1, big1],
+                [key2, big2],
+            ]);
+            deepEqual(await states.insertInitialState(headerHash, entries), Result.ok(OK));
+            const state = states.getState(headerHash);
+            assert.ok(state !== null);
+            assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
+            assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+    it("updates state", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const state = InMemoryState.empty(spec);
+            await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
+            const newState = states.getState(headerHash);
+            assert.ok(newState !== null);
+            const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
+            const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
+            const stateUpdate = {
+                timeslot: tryAsTimeSlot(15),
+                privilegedServices: PrivilegedServices.create({
+                    manager: tryAsServiceId(1),
+                    assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
+                    delegator: tryAsServiceId(3),
+                    registrar: tryAsServiceId(4),
+                    autoAccumulateServices: new Map(),
+                }),
+                updated: new Map([
+                    [
+                        tryAsServiceId(1),
+                        UpdateService.create({
+                            serviceInfo: ServiceAccountInfo.create({
+                                codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
+                                balance: tryAsU64(1_000_000),
+                                accumulateMinGas: tryAsServiceGas(10_000),
+                                onTransferMinGas: tryAsServiceGas(5_000),
+                                storageUtilisationBytes: tryAsU64(1_000),
+                                gratisStorage: tryAsU64(0),
+                                storageUtilisationCount: tryAsU32(1),
+                                created: tryAsTimeSlot(0),
+                                lastAccumulation: tryAsTimeSlot(0),
+                                parentService: tryAsServiceId(0),
+                            }),
+                            lookupHistory,
+                        }),
+                    ],
+                ]),
+            };
+            deepEqual(state.applyUpdate(stateUpdate), Result.ok(OK));
+            deepEqual(await states.updateAndSetState(headerHash2, newState, stateUpdate), Result.ok(OK));
+            const updatedState = states.getState(headerHash2);
+            assert.ok(updatedState !== null);
+            deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
+                [
+                    tryAsServiceId(1),
+                    {
+                        storageKeys: [],
+                        preimages: [],
+                        lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
+                    },
+                ],
+            ])), state);
+            assert.strictEqual(`${await states.getStateRoot(updatedState)}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+    it("imports a more complex state", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const initialState = testState();
+            const initialService = initialState.services.get(tryAsServiceId(0));
+            assert.ok(initialService !== undefined);
+            const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
+            deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
+            const newState = states.getState(headerHash);
+            assert.ok(newState !== null);
+            assert.strictEqual(`${await states.getStateRoot(newState)}`, `${serialized.getRootHash(blake2b)}`);
+            deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+});

package/packages/jam/database-lmdb/hybrid-states.d.ts

@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
  * Reads go straight to lmdb, which keeps its own page cache.
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
  * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
  */
 export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
     private readonly spec;
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     private readonly inMemStates;
     private readonly lmdbValues;
     private readonly valuesDb;
+    private readonly refsStore;
+    private readonly refs;
+    private pendingCleanup;
     static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
         spec: ChainSpec;
         blake2b: Blake2b;
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    private applyRefs;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;
     /** Write new large values to LMDB in one transaction. */

package/packages/jam/database-lmdb/hybrid-states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
+{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-lmdb/hybrid-states.js

@@ -1,9 +1,11 @@
 import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
+import { Logger } from "#@typeberry/logger";
 import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
 import { leafComparator } from "#@typeberry/trie";
 import { OK, Result } from "#@typeberry/utils";
 import { LmdbRoot } from "./root.js";
+const logger = Logger.new(import.meta.filename, "db");
 /**
  * Hybrid serialized-states db.
  *
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
  * Reads go straight to lmdb, which keeps its own page cache.
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
  * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
  */
 export class HybridSerializedStates {
     spec;
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
     lmdbValues;
     // A single shared values accessor reused by every `LeafDb` we hand out.
     valuesDb;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
     static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
         const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
         return new HybridSerializedStates(spec, blake2b, root);
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
             return res;
         }
         this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async updateAndSetState(header, state, update) {
         const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
         // Clone the leaf set before mutating: the previous state keeps using its own.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
         const res = await this.writeValues(values);
         if (res.isError) {
             // Leave the caller's state untouched: its new leaves would reference
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
         // values are durably written.
         state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
         this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
         const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
-        // We only remove the state from memory - values are not pruned at all,
-        // but since they are stored on disk we should be safe.
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.inMemStates.delete(header);
     }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => this.lmdbValues.transaction(() => {
+            for (const v of update.removeValues) {
+                this.lmdbValues.remove(v.raw);
+            }
+        }))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
+    }
     diskSizeInBytes() {
         return this.root.sizeInBytes();
     }
     async close() {
+        await this.pendingCleanup;
         await this.lmdbValues.close();
         await this.root.close();
     }
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
             return Result.ok(OK);
         }
         try {
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
             await this.lmdbValues.transaction(() => {
                 for (const [hash, val] of values) {
                     this.lmdbValues.put(hash.raw, val.raw);

package/packages/jam/database-lmdb/hybrid-states.test.js

@@ -2,12 +2,14 @@
 import assert from "node:assert";
 import * as fs from "node:fs";
 import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { tryAsServiceId } from "#@typeberry/block";
 import { Bytes, BytesBlob } from "#@typeberry/bytes";
 import { tinyChainSpec } from "#@typeberry/config";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
-import { InMemoryState } from "#@typeberry/state";
+import { InMemoryState, StorageItem, UpdateStorage, } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
 import { StateEntries } from "#@typeberry/state-merkleization";
-import { deepEqual, OK, Result } from "#@typeberry/utils";
+import { asOpaqueType, deepEqual, OK, Result } from "#@typeberry/utils";
 import { HybridSerializedStates } from "./hybrid-states.js";
 let blake2b;
 before(async () => {
@@ -86,3 +88,93 @@ describe("Hybrid serialized states", () => {
         }
     });
 });
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+const storageKey = asOpaqueType(BytesBlob.blobFromString("test-key"));
+/** A state update writing a single large (non-embedded) value under `storageKey`. */
+function storageUpdate(value) {
+    const item = StorageItem.create({ key: storageKey, value: BytesBlob.blobFromString(value) });
+    return {
+        storage: new Map([[tryAsServiceId(0), [UpdateStorage.set({ storage: item })]]]),
+    };
+}
+// > 32 bytes => stored in the values db (not embedded in the leaf).
+const BIG_1 = "a".repeat(100);
+const BIG_2 = "b".repeat(100);
+/** `true` if every value referenced by the state can still be resolved. */
+function canReadFully(state) {
+    if (state === null) {
+        return false;
+    }
+    try {
+        state.backend.intoStateEntries();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Value removals are queued, so poll for the expected outcome. */
+async function eventually(check, what, timeoutMs = 5_000) {
+    const start = Date.now();
+    while (!check()) {
+        if (Date.now() - start > timeoutMs) {
+            assert.fail(`Timed out waiting for: ${what}`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+}
+describe("Hybrid serialized states value refcounting", () => {
+    const spec = tinyChainSpec;
+    let dbPath = "";
+    beforeEach(() => {
+        dbPath = createTempDir();
+    });
+    afterEach(() => {
+        fs.rmSync(dbPath, { recursive: true });
+    });
+    it("removes a replaced value from disk once the replacement finalizes", async () => {
+        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a handle to the post-1 state, surviving the pruning below
+            const stale1 = states.getState(hh(1));
+            const s1 = states.getState(hh(1));
+            assert.ok(s1 !== null);
+            await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
+            states.commitFinalized([hh(1)]);
+            assert.ok(canReadFully(stale1), "still referenced by the finalized tip");
+            states.commitFinalized([hh(2)]);
+            await eventually(() => !canReadFully(stale1), "replaced value removed from lmdb");
+            assert.ok(canReadFully(states.getState(hh(2))), "the new finalized tip stays fully readable");
+        }
+        finally {
+            await states.close();
+        }
+    });
+    it("collects values of a pruned dead fork and keeps the surviving chain intact", async () => {
+        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a dead fork on top of genesis, inserting a different value
+            const fork = states.getState(hh(0));
+            assert.ok(fork !== null);
+            await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_2));
+            const staleFork = states.getState(hh(0xaa));
+            states.markUnused(hh(0xaa));
+            assert.strictEqual(states.getState(hh(0xaa)), null);
+            await eventually(() => !canReadFully(staleFork), "fork-only value removed from lmdb");
+            assert.ok(canReadFully(states.getState(hh(1))), "surviving chain is unaffected");
+        }
+        finally {
+            await states.close();
+        }
+    });
+});

package/packages/jam/database-lmdb/states.d.ts

@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
     updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(root: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;

package/packages/jam/database-lmdb/states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
+{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}

package/packages/jam/database-lmdb/states.js

@@ -128,6 +128,11 @@ export class LmdbStates {
         }
         return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
     }
+    commitFinalized(_headers) {
+        // Values are never pruned here. This db survives restarts, so refcounting
+        // would need counts persisted (and crash-consistent) alongside the values.
+        // See the TODO above - not implemented until actually needed.
+    }
     markUnused(header) {
         this.states.removeSync(header.raw);
     }

package/packages/jam/node/export.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBA2F9G"}
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}

package/packages/jam/node/export.js

@@ -3,9 +3,10 @@ import path from "node:path";
 import { Block as BlockCodec } from "#@typeberry/block";
 import { Bytes } from "#@typeberry/bytes";
 import { Encoder } from "#@typeberry/codec";
+import { RegularStateBackend } from "#@typeberry/config-node";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { Logger } from "#@typeberry/logger";
-import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath } from "./common.js";
 export async function exportBlocks(jamNodeConfig, output, withRelPath) {
     const logger = Logger.new(import.meta.filename, "export");
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
     const blake2b = await Blake2b.createHasher();
     const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
     const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
-    const config = LmdbWorkerConfig.new({
-        nodeName: jamNodeConfig.nodeName,
-        chainSpec,
-        blake2b,
-        dbPath,
-        workerParams: undefined,
-    });
-    const rootDb = config.openDatabase();
+    const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
+        ? FjallWorkerConfig.new({
+            nodeName: jamNodeConfig.nodeName,
+            chainSpec,
+            blake2b,
+            dbPath,
+            workerParams: undefined,
+        })
+        : LmdbWorkerConfig.new({
+            nodeName: jamNodeConfig.nodeName,
+            chainSpec,
+            blake2b,
+            dbPath,
+            workerParams: undefined,
+        });
+    const rootDb = await config.openDatabase();
     const blocks = rootDb.getBlocksDb();
     logger.info `📖 Gathering blocks...`;
     const hashes = [];

package/packages/jam/node/main-fuzz.d.ts

@@ -1,5 +1,6 @@
 import { type FuzzVersion } from "#@typeberry/ext-ipc";
 import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
+import { type Closer } from "#@typeberry/utils";
 import type { JamConfig } from "./jam-config.js";
 export type FuzzConfig = {
     version: FuzzVersion;
@@ -24,5 +25,7 @@ export declare function getFuzzDetails(): {
     nodeVersion: fuzzV1.Version;
     gpVersion: fuzzV1.Version;
 };
-export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<() => void>;
+export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<{
+    close: Closer;
+}>;
 //# sourceMappingURL=main-fuzz.d.ts.map

package/packages/jam/node/main-fuzz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAWF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBAqHxF"}
+{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}