@typeberry/lib 0.9.0 → 0.10.0

package/packages/jam/database/leaf-db-update.js

@@ -7,18 +7,34 @@ export function updateLeafs(leafs, blake2b, data) {
     const blake2bTrieHasher = getBlake2bTrieHasher(blake2b);
     // We will collect all values that don't fit directly into leaf nodes.
     const values = [];
+    // Value hashes that are no longer referenced by the leaf they were stored under.
+    // NOTE this does not yet mean they can be removed from the DB - the same value
+    // may still be referenced by another leaf or another state. It's up to value
+    // ref-counting to decide that.
+    const removed = [];
     for (const [action, key, value] of data) {
         if (action === StateEntryUpdateAction.Insert) {
             const leafNode = InMemoryTrie.constructLeaf(blake2bTrieHasher, key.asOpaque(), value);
-            leafs.replace(leafNode);
-            if (!leafNode.hasEmbeddedValue()) {
-                values.push([leafNode.getValueHash(), value]);
+            const displaced = leafs.replace(leafNode);
+            const newHash = leafNode.hasEmbeddedValue() ? null : leafNode.getValueHash();
+            const oldHash = displaced !== undefined && !displaced.hasEmbeddedValue() ? displaced.getValueHash() : null;
+            // Re-inserting the exact same value under the same key is a no-op for the DB.
+            if (newHash !== null && oldHash !== null && newHash.isEqualTo(oldHash)) {
+                continue;
+            }
+            if (newHash !== null) {
+                values.push([newHash, value]);
+            }
+            if (oldHash !== null) {
+                removed.push(oldHash);
             }
         }
         else if (action === StateEntryUpdateAction.Remove) {
             const leafNode = InMemoryTrie.constructLeaf(blake2bTrieHasher, key.asOpaque(), BytesBlob.empty());
-            leafs.removeOne(leafNode);
-            // TODO [ToDr] Handle ref-counting values or updating some header-hash-based references.
+            const removedLeaf = leafs.removeOne(leafNode);
+            if (removedLeaf !== undefined && !removedLeaf.hasEmbeddedValue()) {
+                removed.push(removedLeaf.getValueHash());
+            }
         }
         else {
             assertNever(action);
@@ -26,6 +42,7 @@ export function updateLeafs(leafs, blake2b, data) {
     }
     return {
         values,
+        removed,
         leafs,
     };
 }

package/packages/jam/database/leaf-db-update.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=leaf-db-update.test.d.ts.map

package/packages/jam/database/leaf-db-update.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"leaf-db-update.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/leaf-db-update.test.ts"],"names":[],"mappings":""}

package/packages/jam/database/leaf-db-update.test.js

@@ -0,0 +1,81 @@
+import assert from "node:assert";
+import { before, describe, it } from "node:test";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { SortedSet } from "#@typeberry/collections";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
+import { StateEntryUpdateAction } from "#@typeberry/state-merkleization";
+import { leafComparator } from "#@typeberry/trie";
+import { updateLeafs } from "./leaf-db-update.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+function key(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+/** A value larger than 32 bytes, so it does not fit into the leaf and is hash-referenced. */
+function bigValue(b) {
+    return Bytes.fill(64, b);
+}
+/** A value that fits directly into the leaf node. */
+function smallValue(b) {
+    return Bytes.fill(4, b);
+}
+function emptyLeafs() {
+    return SortedSet.fromArray(leafComparator, []);
+}
+function asStrings(hashes) {
+    return hashes.map((h) => h.toString());
+}
+describe("updateLeafs", () => {
+    it("collects a freshly inserted non-embedded value and removes nothing", () => {
+        const leafs = emptyLeafs();
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        assert.strictEqual(res.values.length, 1);
+        assert.deepStrictEqual(res.removed, []);
+    });
+    it("collects nothing for an embedded value", () => {
+        const leafs = emptyLeafs();
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), smallValue(0xaa)]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(res.removed, []);
+    });
+    it("emits the displaced value hash in `removed` when a leaf is replaced", () => {
+        const leafs = emptyLeafs();
+        const first = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const oldHash = first.values[0][0];
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xbb)]]);
+        assert.strictEqual(res.values.length, 1);
+        assert.deepStrictEqual(asStrings(res.removed), asStrings([oldHash]));
+    });
+    it("cancels out when the same value is re-inserted at the same key", () => {
+        const leafs = emptyLeafs();
+        updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(res.removed, []);
+    });
+    it("emits the old hash when a non-embedded value becomes embedded", () => {
+        const leafs = emptyLeafs();
+        const first = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const oldHash = first.values[0][0];
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), smallValue(0xbb)]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(asStrings(res.removed), asStrings([oldHash]));
+    });
+    it("emits the removed value hash when a non-embedded leaf is removed", () => {
+        const leafs = emptyLeafs();
+        const first = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const oldHash = first.values[0][0];
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Remove, key(1), BytesBlob.empty()]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(asStrings(res.removed), asStrings([oldHash]));
+    });
+    it("removes nothing when an embedded leaf is removed", () => {
+        const leafs = emptyLeafs();
+        updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), smallValue(0xaa)]]);
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Remove, key(1), BytesBlob.empty()]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(res.removed, []);
+    });
+});

package/packages/jam/database/serialized-states-db.d.ts

@@ -15,6 +15,8 @@ export declare class InMemorySerializedStates implements StatesDb<SerializedStat
     private readonly blake2b;
     private readonly db;
     private readonly valuesDb;
+    private readonly refsStore;
+    private readonly refs;
     static new({ chainSpec }: {
         chainSpec: ChainSpec;
     }): Promise<InMemorySerializedStates>;
@@ -27,7 +29,9 @@ export declare class InMemorySerializedStates implements StatesDb<SerializedStat
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
+    private applyRefs;
     close(): Promise<void>;
 }
 //# sourceMappingURL=serialized-states-db.d.ts.map

package/packages/jam/database/serialized-states-db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"serialized-states-db.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/serialized-states-db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE5E,qCAAqC;AACrC,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;AAEhG,sCAAsC;AACtC,qBAAa,wBAAyB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAc1G,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAd1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAyE;IAC5F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8D;WAE1E,GAAG,CAAC,EAAE,SAAS,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAA;KAAE;IAKxD,MAAM,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAIpF,OAAO;IAKD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAiBxG,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAIpE,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAsBxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAkB5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
+{"version":3,"file":"serialized-states-db.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/serialized-states-db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAG5E,qCAAqC;AACrC,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;AAEhG,sCAAsC;AACtC,qBAAa,wBAAyB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAgB1G,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAhB1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAyE;IAC5F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8D;IACvF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;WAEzC,GAAG,CAAC,EAAE,SAAS,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAA;KAAE;IAKxD,MAAM,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAIpF,OAAO;IAKD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBxG,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAIpE,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAwBxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAkB5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,OAAO,CAAC,SAAS;IAOX,KAAK;CACZ"}

package/packages/jam/database/serialized-states-db.js

@@ -5,12 +5,15 @@ import { leafComparator } from "#@typeberry/trie";
 import { OK, Result } from "#@typeberry/utils";
 import { LeafDb } from "./leaf-db.js";
 import { updateLeafs } from "./leaf-db-update.js";
+import { InMemoryValueRefsStore, ValueRefs } from "./value-refs.js";
 /** In-memory serialized-states db. */
 export class InMemorySerializedStates {
     spec;
     blake2b;
     db = HashDictionary.new();
     valuesDb = HashDictionary.new();
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
     static async new({ chainSpec }) {
         const blake2b = await Blake2b.createHasher();
         return new InMemorySerializedStates(chainSpec, blake2b);
@@ -30,6 +33,7 @@ export class InMemorySerializedStates {
             this.valuesDb.set(val[0], val[1]);
         }
         this.db.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -40,17 +44,19 @@ export class InMemorySerializedStates {
         const updatedValues = serializeStateUpdate(this.spec, blake2b, update);
         // make sure to clone the leafs before writing, since the collection is re-used.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, blake2b, updatedValues);
         // make sure to reset the cache and re-create leafsdb lookup
         state.updateBackend(LeafDb.fromLeaves(leafs, state.backend.db));
         // insert values to the db
         // valuesdb can be shared between all states because it's just
-        // <valuehash> -> <value> mapping and existence is managed by trie leafs.
+        // <valuehash> -> <value> mapping and existence is managed by value
+        // refcounting, driven by the per-block deltas recorded below.
         for (const val of values) {
             this.valuesDb.set(val[0], val[1]);
         }
         // store new set of leaves
         this.db.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     getState(header) {
@@ -70,8 +76,18 @@ export class InMemorySerializedStates {
         });
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.db.delete(header);
     }
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        for (const v of update.removeValues) {
+            this.valuesDb.delete(v);
+        }
+    }
     async close() { }
 }

package/packages/jam/database/serialized-states-db.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=serialized-states-db.test.d.ts.map

package/packages/jam/database/serialized-states-db.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialized-states-db.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/serialized-states-db.test.ts"],"names":[],"mappings":""}

package/packages/jam/database/serialized-states-db.test.js

@@ -0,0 +1,109 @@
+import assert from "node:assert";
+import { before, describe, it } from "node:test";
+import { tryAsServiceId } from "#@typeberry/block";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { tinyChainSpec } from "#@typeberry/config";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
+import { StorageItem, UpdateStorage } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
+import { StateEntries } from "#@typeberry/state-merkleization";
+import { asOpaqueType, deepEqual, OK, Result } from "#@typeberry/utils";
+import { InMemorySerializedStates } from "./serialized-states-db.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+const spec = tinyChainSpec;
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+const storageKey = asOpaqueType(BytesBlob.blobFromString("test-key"));
+/** A state update writing a single large (non-embedded) value under `storageKey`. */
+function storageUpdate(value) {
+    const item = StorageItem.create({ key: storageKey, value: BytesBlob.blobFromString(value) });
+    return {
+        storage: new Map([[tryAsServiceId(0), [UpdateStorage.set({ storage: item })]]]),
+    };
+}
+// > 32 bytes => stored in the values db (not embedded in the leaf).
+const BIG_1 = "a".repeat(100);
+const BIG_2 = "b".repeat(100);
+async function newStates() {
+    const states = InMemorySerializedStates.withHasher({ chainSpec: spec, blake2b });
+    const res = await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+    deepEqual(res, Result.ok(OK));
+    return states;
+}
+/** `true` if every value referenced by the state can still be resolved. */
+function canReadFully(state) {
+    if (state === null) {
+        return false;
+    }
+    try {
+        state.backend.intoStateEntries();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+describe("InMemorySerializedStates value refcounting", () => {
+    it("keeps a replaced value until the block replacing it finalizes", async () => {
+        const states = await newStates();
+        const s0 = states.getState(hh(0));
+        assert.ok(s0 !== null);
+        await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+        // a handle to the post-1 state, surviving the pruning below
+        const stale1 = states.getState(hh(1));
+        const s1 = states.getState(hh(1));
+        assert.ok(s1 !== null);
+        await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
+        states.commitFinalized([hh(1)]);
+        assert.ok(canReadFully(stale1), "still referenced by the finalized tip");
+        states.commitFinalized([hh(2)]);
+        assert.ok(!canReadFully(stale1), "replaced value is dropped once the replacement finalizes");
+        assert.ok(canReadFully(states.getState(hh(2))), "the new finalized tip stays fully readable");
+    });
+    it("collects values of a pruned dead fork and keeps the surviving chain intact", async () => {
+        const states = await newStates();
+        const s0 = states.getState(hh(0));
+        assert.ok(s0 !== null);
+        await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+        // a dead fork on top of genesis, inserting a different value
+        const fork = states.getState(hh(0));
+        assert.ok(fork !== null);
+        await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_2));
+        const staleFork = states.getState(hh(0xaa));
+        states.markUnused(hh(0xaa));
+        assert.strictEqual(states.getState(hh(0xaa)), null);
+        assert.ok(!canReadFully(staleFork), "fork-only value is collected");
+        assert.ok(canReadFully(states.getState(hh(1))), "surviving chain is unaffected");
+    });
+    it("does not collect a value shared with a pruned fork", async () => {
+        const states = await newStates();
+        const s0 = states.getState(hh(0));
+        assert.ok(s0 !== null);
+        await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+        // the fork writes the very same value
+        const fork = states.getState(hh(0));
+        assert.ok(fork !== null);
+        await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_1));
+        states.markUnused(hh(0xaa));
+        assert.ok(canReadFully(states.getState(hh(1))), "value still referenced by the surviving chain");
+    });
+    it("follows the importer lifecycle: commit finalized, then prune", async () => {
+        const states = await newStates();
+        const s0 = states.getState(hh(0));
+        assert.ok(s0 !== null);
+        await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+        const s1 = states.getState(hh(1));
+        assert.ok(s1 !== null);
+        await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
+        // finality round: 1 and 2 finalized, genesis and 1 pruned
+        states.commitFinalized([hh(1), hh(2)]);
+        states.markUnused(hh(0));
+        states.markUnused(hh(1));
+        assert.strictEqual(states.getState(hh(1)), null);
+        assert.ok(canReadFully(states.getState(hh(2))), "finalized tip fully readable after pruning");
+    });
+});

package/packages/jam/database/states.d.ts

@@ -33,6 +33,16 @@ export interface StatesDb<T extends State = State> {
     updateAndSetState(header: HeaderHash, state: T, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     /** Retrieve posterior state of given header. */
     getState(header: HeaderHash): T | null;
+    /**
+     * Notify the backend about newly finalized blocks, in ancestor-first order.
+     *
+     * Backends that share data between states (e.g. a content-addressed values DB)
+     * use this to move the blocks' references from speculative to finalized.
+     * MUST be called before `markUnused` of the same finality round, otherwise
+     * the newly finalized blocks would be treated as dead forks and values still
+     * referenced by the finalized state could be dropped.
+     */
+    commitFinalized(headers: HeaderHash[]): void;
     /** Mark state as no longer needed. Backend may remove it asynchronously. */
     markUnused(header: HeaderHash): void;
     /**
@@ -56,6 +66,7 @@ export declare class InMemoryStates implements StatesDb<InMemoryState> {
     /** Insert a full state into the database. */
     insertInitialState(headerHash: HeaderHash, state: InMemoryState): Promise<Result<OK, StateUpdateError>>;
     getState(headerHash: HeaderHash): InMemoryState | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     close(): Promise<void>;
 }

package/packages/jam/database/states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
+{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC;;;;;;;;OAQG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE7C,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAI7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}

package/packages/jam/database/states.js

@@ -53,6 +53,9 @@ export class InMemoryStates {
         }
         return InMemoryState.copyFrom(this.spec, state, state.intoServicesData());
     }
+    commitFinalized(_headers) {
+        // nothing to do: every state is a full, independent copy.
+    }
     markUnused(header) {
         this.db.delete(header);
     }

package/packages/jam/database/value-refs.d.ts

@@ -0,0 +1,117 @@
+import type { HeaderHash } from "#@typeberry/block";
+import type { ValueHash } from "#@typeberry/trie";
+/** The value hashes a single block introduced and stopped referencing. */
+export interface ValueDelta {
+    /** Value hashes (non-embedded) that the block started referencing. */
+    inserted: ValueHash[];
+    /** Value hashes (non-embedded) that the block stopped referencing. */
+    removed: ValueHash[];
+}
+/**
+ * Synchronous, read-only view of the persisted refcounting state.
+ *
+ * Both LMDB and fjall offer synchronous reads, so reads can go straight
+ * to the backing store. All writes go through `ValueRefsUpdate` batches
+ * instead, since persistent backends can only write asynchronously
+ * (LMDB transactions, fjall inserts + persist).
+ */
+export interface ValueRefsReader {
+    /** How many leaves of the finalized-tip state reference the value. Missing keys read as `0`. */
+    getFinalizedCount(hash: ValueHash): number;
+    /** How many surviving, not-yet-finalized blocks inserted the value. Missing keys read as `0`. */
+    getPendingCount(hash: ValueHash): number;
+    /** The value delta of a not-yet-finalized block, if known. */
+    getDelta(header: HeaderHash): ValueDelta | undefined;
+}
+/**
+ * A batch of refcounting mutations produced by a single `ValueRefs` operation.
+ *
+ * The backend is responsible for applying the batch using its own write
+ * primitive - ideally atomically with the state write that triggered it
+ * (one LMDB transaction, one fjall persist).
+ *
+ * Counts are absolute values rather than increments, so applying the same
+ * update more than once (e.g. on crash-replay) is harmless.
+ *
+ * When atomicity is not available, apply in field order and `removeValues`
+ * strictly last: a crash after counts are persisted but before values are
+ * removed only leaks values, while the opposite order could drop a value
+ * the persisted counts still consider referencd.
+ */
+export interface ValueRefsUpdate {
+    /** New absolute finalized counts. Count `0` means the entry should be removed. */
+    finalizedCounts: [ValueHash, number][];
+    /** New absolute pending counts. Count `0` means the entry should be removed. */
+    pendingCounts: [ValueHash, number][];
+    /** Deltas of freshly imported blocks to persist. */
+    setDeltas: [HeaderHash, ValueDelta][];
+    /** Deltas consumed by finalization or fork pruning. */
+    removeDeltas: HeaderHash[];
+    /** Values that are no longer referenced and can be removed from the values DB. */
+    removeValues: ValueHash[];
+}
+/** `true` if applying the update would not change anything. */
+export declare function isEmptyUpdate(update: ValueRefsUpdate): boolean;
+/**
+ * Decides when a content-addressed value can be removed from the values DB.
+ *
+ * A value is needed as long as some surviving state references it. Surviving
+ * states are the finalized tip plus its unfinalized descendants, so we track:
+ * - `finalized`: how many leaves in the current finalized-tip state reference V,
+ *   advanced strictly by replaying finalized blocks' deltas (never on prune);
+ * - `pending`: how many surviving, not-yet-finalized blocks inserted V (an
+ *   over-approximation of unfinalized references).
+ *
+ * A value is removable exactly when both counts reach zero.
+ *
+ * This class only makes decisions: every operation reads the current state
+ * through `ValueRefsReader` and returns a `ValueRefsUpdate` describing what
+ * should change. Nothing is written here - the owning backend applies the
+ * update with whatever consistency guarantees it can provide.
+ */
+export declare class ValueRefs {
+    private readonly reader;
+    constructor(reader: ValueRefsReader);
+    /** Record values referenced by the genesis / initial finalized state. */
+    onInitial(inserted: ValueHash[]): ValueRefsUpdate;
+    /**
+     * Record a freshly imported, not-yet-finalized block.
+     *
+     * Importing the same header twice is a no-op, otherwise the second import
+     * would double-count `pending` references and pin the values forever.
+     */
+    onImport(header: HeaderHash, delta: ValueDelta): ValueRefsUpdate;
+    /**
+     * Apply the value deltas of newly finalized blocks, in finalized (ancestor-first) order.
+     *
+     * Moves each inserted value from `pending` to `finalized`, drops `finalized`
+     * references for removed values, and collects anything that becomes unreferenced.
+     */
+    commitFinalized(headers: HeaderHash[]): ValueRefsUpdate;
+    /**
+     * Release the speculative references of a state being discarded.
+     *
+     * For a header that was still unfinalized (a dead fork) the returned update
+     * releases its inserted values, which may become collectable. For an already
+     * finalized state the update is empty (check with `isEmptyUpdate`): its delta
+     * was consumed on finality and its values are accounted for in `finalized`.
+     */
+    releaseUnfinalized(header: HeaderHash): ValueRefsUpdate;
+}
+/**
+ * In-memory refcounting store, reusable by the in-memory and hybrid states DBs
+ * (the hybrids cannot resume from disk anyway, so persisting counts buys nothing).
+ *
+ * NOTE: `apply` does not touch the values DB - the caller owns it and must
+ * handle `update.removeValues` itself.
+ */
+export declare class InMemoryValueRefsStore implements ValueRefsReader {
+    private readonly finalized;
+    private readonly pending;
+    private readonly deltas;
+    getFinalizedCount(hash: ValueHash): number;
+    getPendingCount(hash: ValueHash): number;
+    getDelta(header: HeaderHash): ValueDelta | undefined;
+    apply(update: ValueRefsUpdate): void;
+}
+//# sourceMappingURL=value-refs.d.ts.map

package/packages/jam/database/value-refs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"value-refs.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/value-refs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,sEAAsE;IACtE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,sEAAsE;IACtE,OAAO,EAAE,SAAS,EAAE,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B,gGAAgG;IAChG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3C,iGAAiG;IACjG,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IACzC,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;CACtD;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,eAAe;IAC9B,kFAAkF;IAClF,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACvC,gFAAgF;IAChF,aAAa,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACrC,oDAAoD;IACpD,SAAS,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;IACtC,uDAAuD;IACvD,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,kFAAkF;IAClF,YAAY,EAAE,SAAS,EAAE,CAAC;CAC3B;AAED,+DAA+D;AAC/D,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAQ9D;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,eAAe;IAEpD,yEAAyE;IACzE,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,eAAe;IAQjD;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,eAAe;IAYhE;;;;;OAKG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,eAAe;IAoBvD;;;;;;;OAOG;IACH,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe;CAYxD;AA0ED;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,eAAe;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2D;IACrF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2D;IACnF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgE;IAEvF,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAI1C,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS;IAIpD,KAAK,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;CAUrC"}