npm - @typeberry/lib - Versions diffs - 0.9.0-59ce083 → 0.9.0-6850f84 - Mend

@typeberry/lib 0.9.0-59ce083 → 0.9.0-6850f84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (39) hide show

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@typeberry/lib",
-  "version": "0.9.0-59ce083",
+  "version": "0.9.0-6850f84",
   "description": "Typeberry Library",
   "main": "./bin/lib/index.js",
   "types": "./bin/lib/index.d.ts",

package/packages/core/collections/sorted-array.d.ts CHANGED Viewed

@@ -56,7 +56,7 @@ export declare class SortedArray<V> implements ImmutableSortedArray<V> {
     findIndex(v: V): number;
     findExact(v: V): V | undefined;
     /** Remove one matching element from the collection. */
-    removeOne(v: V): void;
+    removeOne(v: V): V | undefined;
     has(v: V): boolean;
     get length(): number;
     /** Return a regular array that's a copy of this one. */

package/packages/core/collections/sorted-array.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sorted-array.d.ts","sourceRoot":"","sources":["../../../../../packages/core/collections/sorted-array.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGtD,MAAM,WAAW,oBAAoB,CAAC,CAAC,CAAE,SAAQ,QAAQ,CAAC,CAAC,CAAC;IAC1D,wBAAwB;IACxB,IAAI,KAAK,IAAI,SAAS,CAAC,EAAE,CAAC;IAC1B,6BAA6B;IAC7B,IAAI,UAAU,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAEhC;;;OAGG;IACH,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;IAExB;;;;;;;OAOG;IACH,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;IAE/B,qDAAqD;IACrD,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IAEnB,+CAA+C;IAC/C,IAAI,MAAM,IAAI,MAAM,CAAC;IAErB,wDAAwD;IACxD,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;CAC1C;AAED;;;;;;;GAOG;AACH,qBAAa,WAAW,CAAC,CAAC,CAAE,YAAW,oBAAoB,CAAC,CAAC,CAAC;aAgC1C,KAAK,EAAE,CAAC,EAAE;aACV,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IAhC3C;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAMvE;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAkB7E,SAAS,aACS,KAAK,EAAE,CAAC,EAAE,EACV,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IAG3C,4CAA4C;IACrC,MAAM,CAAC,CAAC,EAAE,CAAC;IAKlB;;;;OAIG;IACI,GAAG,IAAI,CAAC,GAAG,SAAS;IAIpB,SAAS,CAAC,CAAC,EAAE,CAAC;IASd,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS;IASrC,uDAAuD;IAChD,SAAS,CAAC,CAAC,EAAE,CAAC;~~IAQd~~,GAAG,CAAC,CAAC,EAAE,CAAC;IAIf,IAAW,MAAM,IAAI,MAAM,CAE1B;IAED,wDAAwD;IACjD,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAI/C,SAAS,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;;;;IA8B3B,4DAA4D;IAC5D,MAAM,CAAC,wBAAwB,CAAC,CAAC,EAAE,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC;IAqClG,kDAAkD;IACjD,CAAC,MAAM,CAAC,QAAQ,CAAC;CAKnB"}
1	+ {"version":3,"file":"sorted-array.d.ts","sourceRoot":"","sources":["../../../../../packages/core/collections/sorted-array.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGtD,MAAM,WAAW,oBAAoB,CAAC,CAAC,CAAE,SAAQ,QAAQ,CAAC,CAAC,CAAC;IAC1D,wBAAwB;IACxB,IAAI,KAAK,IAAI,SAAS,CAAC,EAAE,CAAC;IAC1B,6BAA6B;IAC7B,IAAI,UAAU,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAEhC;;;OAGG;IACH,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;IAExB;;;;;;;OAOG;IACH,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;IAE/B,qDAAqD;IACrD,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IAEnB,+CAA+C;IAC/C,IAAI,MAAM,IAAI,MAAM,CAAC;IAErB,wDAAwD;IACxD,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;CAC1C;AAED;;;;;;;GAOG;AACH,qBAAa,WAAW,CAAC,CAAC,CAAE,YAAW,oBAAoB,CAAC,CAAC,CAAC;aAgC1C,KAAK,EAAE,CAAC,EAAE;aACV,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IAhC3C;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAMvE;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAkB7E,SAAS,aACS,KAAK,EAAE,CAAC,EAAE,EACV,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IAG3C,4CAA4C;IACrC,MAAM,CAAC,CAAC,EAAE,CAAC;IAKlB;;;;OAIG;IACI,GAAG,IAAI,CAAC,GAAG,SAAS;IAIpB,SAAS,CAAC,CAAC,EAAE,CAAC;IASd,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS;IASrC,uDAAuD;IAChD,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS;IAU9B,GAAG,CAAC,CAAC,EAAE,CAAC;IAIf,IAAW,MAAM,IAAI,MAAM,CAE1B;IAED,wDAAwD;IACjD,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAI/C,SAAS,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;;;;IA8B3B,4DAA4D;IAC5D,MAAM,CAAC,wBAAwB,CAAC,CAAC,EAAE,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC;IAqClG,kDAAkD;IACjD,CAAC,MAAM,CAAC,QAAQ,CAAC;CAKnB"}

package/packages/core/collections/sorted-array.js CHANGED Viewed

@@ -69,9 +69,11 @@ export class SortedArray {
     removeOne(v) {
         const findIdx = this.binarySearch(v);
         if (findIdx.isEqual) {
-            // remove the element
-            this.array.splice(findIdx.idx, 1);
+            // remove the element and return the stored one
+            const [removed] = this.array.splice(findIdx.idx, 1);
+            return removed;
         }
+        return undefined;
     }
     has(v) {
         return this.binarySearch(v).isEqual;

package/packages/core/collections/sorted-array.test.js CHANGED Viewed

@@ -59,6 +59,16 @@ describe("SortedArray", { timeout: 10 }, () => {
         data.removeOne(2);
         assert.deepStrictEqual(data.slice(), [3]);
     });
+    it("should return the stored element that was removed", () => {
+        const byKey = (self, other) => cmp(self[0], other[0]);
+        const data = SortedArray.fromArray(byKey, [
+            [1, "a"],
+            [2, "b"],
+        ]);
+        const removed = data.removeOne([1, "x"]);
+        assert.deepStrictEqual(removed, [1, "a"]);
+        assert.strictEqual(data.removeOne([5, "z"]), undefined);
+    });
     it("should throw when using fromSortedArray and array is not sorted", () => {
         const data = [1, 3, 2];
         const tryToCreate = () => SortedArray.fromSortedArray(cmp, data);

package/packages/core/collections/sorted-set.d.ts CHANGED Viewed

@@ -34,7 +34,7 @@ export declare class SortedSet<V> extends SortedArray<V> implements ImmutableSor
      *
      * Putting another value that's equal via comparator will replace the current one.
      */
-    replace(v: V): void;
+    replace(v: V): V | undefined;
     /** Create a new SortedSet from two sorted collections. */
     static fromTwoSortedCollections<V>(first: ImmutableSortedArray<V>, second: ImmutableSortedArray<V>): SortedSet<V>;
 }

package/packages/core/collections/sorted-set.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sorted-set.d.ts","sourceRoot":"","sources":["../../../../../packages/core/collections/sorted-set.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,KAAK,oBAAoB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3E,MAAM,WAAW,kBAAkB,CAAC,CAAC,CAAE,SAAQ,oBAAoB,CAAC,CAAC,CAAC;CAAG;AAEzE;;;;;;;GAOG;AACH,qBAAa,SAAS,CAAC,CAAC,CAAE,SAAQ,WAAW,CAAC,CAAC,CAAE,YAAW,kBAAkB,CAAC,CAAC,CAAC;IAC/E;;;OAGG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAkBvE;;;OAGG;IACH,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE;IAaxE;;;;OAIG;IACH,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAiB7E,wEAAwE;IACjE,MAAM,CAAC,CAAC,EAAE,CAAC;IAOlB;;;;OAIG;IACI,OAAO,CAAC,CAAC,EAAE,CAAC;~~IAMnB~~,0DAA0D;IAC1D,MAAM,CAAC,wBAAwB,CAAC,CAAC,EAAE,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC;CA0BnG"}
1	+ {"version":3,"file":"sorted-set.d.ts","sourceRoot":"","sources":["../../../../../packages/core/collections/sorted-set.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,KAAK,oBAAoB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3E,MAAM,WAAW,kBAAkB,CAAC,CAAC,CAAE,SAAQ,oBAAoB,CAAC,CAAC,CAAC;CAAG;AAEzE;;;;;;;GAOG;AACH,qBAAa,SAAS,CAAC,CAAC,CAAE,SAAQ,WAAW,CAAC,CAAC,CAAE,YAAW,kBAAkB,CAAC,CAAC,CAAC;IAC/E;;;OAGG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAkBvE;;;OAGG;IACH,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE;IAaxE;;;;OAIG;IACH,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,GAAE,SAAS,CAAC,EAAO;IAiB7E,wEAAwE;IACjE,MAAM,CAAC,CAAC,EAAE,CAAC;IAOlB;;;;OAIG;IACI,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS;IAUnC,0DAA0D;IAC1D,MAAM,CAAC,wBAAwB,CAAC,CAAC,EAAE,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC;CA0BnG"}

package/packages/core/collections/sorted-set.js CHANGED Viewed

@@ -73,8 +73,12 @@ export class SortedSet extends SortedArray {
      */
     replace(v) {
         const findIdx = this.binarySearch(v);
-        const toRemove = findIdx.isEqual ? 1 : 0;
-        this.array.splice(findIdx.idx, toRemove, v);
+        if (findIdx.isEqual) {
+            const [displaced] = this.array.splice(findIdx.idx, 1, v);
+            return displaced;
+        }
+        this.array.splice(findIdx.idx, 0, v);
+        return undefined;
     }
     /** Create a new SortedSet from two sorted collections. */
     static fromTwoSortedCollections(first, second) {

package/packages/core/collections/sorted-set.test.js CHANGED Viewed

@@ -65,6 +65,30 @@ describe("SortedSet", () => {
             assert.deepStrictEqual(data.length, 3);
         });
     });
+    describe("replace", () => {
+        const byKey = (self, other) => cmp(self[0], other[0]);
+        it("should return the displaced element when replacing an equal one", () => {
+            const data = SortedSet.fromArrayUnique(byKey, [
+                [1, "a"],
+                [2, "b"],
+            ]);
+            const displaced = data.replace([1, "c"]);
+            assert.deepStrictEqual(displaced, [1, "a"]);
+            assert.deepStrictEqual(data.slice(), [
+                [1, "c"],
+                [2, "b"],
+            ]);
+        });
+        it("should return undefined when inserting a fresh element", () => {
+            const data = SortedSet.fromArrayUnique(byKey, [[1, "a"]]);
+            const displaced = data.replace([2, "b"]);
+            assert.strictEqual(displaced, undefined);
+            assert.deepStrictEqual(data.slice(), [
+                [1, "a"],
+                [2, "b"],
+            ]);
+        });
+    });
     describe("fromTwoSortedCollections", () => {
         it("should merge two sorted sets without duplicates", () => {
             const arr1 = [1, 2, 3];

package/packages/jam/database/index.d.ts CHANGED Viewed

@@ -4,4 +4,5 @@ export * from "./leaf-db-update.js";
 export * from "./root.js";
 export * from "./serialized-states-db.js";
 export * from "./states.js";
+export * from "./value-refs.js";
 //# sourceMappingURL=index.d.ts.map

package/packages/jam/database/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,aAAa,CAAC"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC"}

package/packages/jam/database/index.js CHANGED Viewed

@@ -4,3 +4,4 @@ export * from "./leaf-db-update.js";
 export * from "./root.js";
 export * from "./serialized-states-db.js";
 export * from "./states.js";
+export * from "./value-refs.js";

package/packages/jam/database/leaf-db-update.d.ts CHANGED Viewed

@@ -5,6 +5,7 @@ import { StateEntryUpdateAction, type StateKey } from "#@typeberry/state-merklei
 import { type LeafNode, type ValueHash } from "#@typeberry/trie";
 export declare function updateLeafs(leafs: SortedSet<LeafNode>, blake2b: Blake2b, data: Iterable<[StateEntryUpdateAction, StateKey | TruncatedHash, BytesBlob]>): {
     values: [ValueHash, BytesBlob][];
+    removed: ValueHash[];
     leafs: SortedSet<LeafNode>;
 };
 //# sourceMappingURL=leaf-db-update.d.ts.map

package/packages/jam/database/leaf-db-update.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"leaf-db-update.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/leaf-db-update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,KAAK,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AACvF,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI9E,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,EAC1B,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,QAAQ,CAAC,CAAC,sBAAsB,EAAE,QAAQ,GAAG,aAAa,EAAE,SAAS,CAAC,CAAC,GAC5E;IACD,MAAM,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;IACjC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;CAC5B,~~CAwBA~~"}
1	+ {"version":3,"file":"leaf-db-update.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/leaf-db-update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,KAAK,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AACvF,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI9E,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,EAC1B,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,QAAQ,CAAC,CAAC,sBAAsB,EAAE,QAAQ,GAAG,aAAa,EAAE,SAAS,CAAC,CAAC,GAC5E;IACD,MAAM,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;IACjC,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;CAC5B,CAyCA"}

package/packages/jam/database/leaf-db-update.js CHANGED Viewed

@@ -7,18 +7,34 @@ export function updateLeafs(leafs, blake2b, data) {
     const blake2bTrieHasher = getBlake2bTrieHasher(blake2b);
     // We will collect all values that don't fit directly into leaf nodes.
     const values = [];
+    // Value hashes that are no longer referenced by the leaf they were stored under.
+    // NOTE this does not yet mean they can be removed from the DB - the same value
+    // may still be referenced by another leaf or another state. It's up to value
+    // ref-counting to decide that.
+    const removed = [];
     for (const [action, key, value] of data) {
         if (action === StateEntryUpdateAction.Insert) {
             const leafNode = InMemoryTrie.constructLeaf(blake2bTrieHasher, key.asOpaque(), value);
-            leafs.replace(leafNode);
-            if (!leafNode.hasEmbeddedValue()) {
-                values.push([leafNode.getValueHash(), value]);
+            const displaced = leafs.replace(leafNode);
+            const newHash = leafNode.hasEmbeddedValue() ? null : leafNode.getValueHash();
+            const oldHash = displaced !== undefined && !displaced.hasEmbeddedValue() ? displaced.getValueHash() : null;
+            // Re-inserting the exact same value under the same key is a no-op for the DB.
+            if (newHash !== null && oldHash !== null && newHash.isEqualTo(oldHash)) {
+                continue;
+            }
+            if (newHash !== null) {
+                values.push([newHash, value]);
+            }
+            if (oldHash !== null) {
+                removed.push(oldHash);
             }
         }
         else if (action === StateEntryUpdateAction.Remove) {
             const leafNode = InMemoryTrie.constructLeaf(blake2bTrieHasher, key.asOpaque(), BytesBlob.empty());
-            leafs.removeOne(leafNode);
-            // TODO [ToDr] Handle ref-counting values or updating some header-hash-based references.
+            const removedLeaf = leafs.removeOne(leafNode);
+            if (removedLeaf !== undefined && !removedLeaf.hasEmbeddedValue()) {
+                removed.push(removedLeaf.getValueHash());
+            }
         }
         else {
             assertNever(action);
@@ -26,6 +42,7 @@ export function updateLeafs(leafs, blake2b, data) {
     }
     return {
         values,
+        removed,
         leafs,
     };
 }

package/packages/jam/database/leaf-db-update.test.d.ts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=leaf-db-update.test.d.ts.map

package/packages/jam/database/leaf-db-update.test.d.ts.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"leaf-db-update.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/leaf-db-update.test.ts"],"names":[],"mappings":""}

package/packages/jam/database/leaf-db-update.test.js ADDED Viewed

@@ -0,0 +1,81 @@
+import assert from "node:assert";
+import { before, describe, it } from "node:test";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { SortedSet } from "#@typeberry/collections";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
+import { StateEntryUpdateAction } from "#@typeberry/state-merkleization";
+import { leafComparator } from "#@typeberry/trie";
+import { updateLeafs } from "./leaf-db-update.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+function key(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+/** A value larger than 32 bytes, so it does not fit into the leaf and is hash-referenced. */
+function bigValue(b) {
+    return Bytes.fill(64, b);
+}
+/** A value that fits directly into the leaf node. */
+function smallValue(b) {
+    return Bytes.fill(4, b);
+}
+function emptyLeafs() {
+    return SortedSet.fromArray(leafComparator, []);
+}
+function asStrings(hashes) {
+    return hashes.map((h) => h.toString());
+}
+describe("updateLeafs", () => {
+    it("collects a freshly inserted non-embedded value and removes nothing", () => {
+        const leafs = emptyLeafs();
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        assert.strictEqual(res.values.length, 1);
+        assert.deepStrictEqual(res.removed, []);
+    });
+    it("collects nothing for an embedded value", () => {
+        const leafs = emptyLeafs();
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), smallValue(0xaa)]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(res.removed, []);
+    });
+    it("emits the displaced value hash in `removed` when a leaf is replaced", () => {
+        const leafs = emptyLeafs();
+        const first = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const oldHash = first.values[0][0];
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xbb)]]);
+        assert.strictEqual(res.values.length, 1);
+        assert.deepStrictEqual(asStrings(res.removed), asStrings([oldHash]));
+    });
+    it("cancels out when the same value is re-inserted at the same key", () => {
+        const leafs = emptyLeafs();
+        updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(res.removed, []);
+    });
+    it("emits the old hash when a non-embedded value becomes embedded", () => {
+        const leafs = emptyLeafs();
+        const first = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const oldHash = first.values[0][0];
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), smallValue(0xbb)]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(asStrings(res.removed), asStrings([oldHash]));
+    });
+    it("emits the removed value hash when a non-embedded leaf is removed", () => {
+        const leafs = emptyLeafs();
+        const first = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), bigValue(0xaa)]]);
+        const oldHash = first.values[0][0];
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Remove, key(1), BytesBlob.empty()]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(asStrings(res.removed), asStrings([oldHash]));
+    });
+    it("removes nothing when an embedded leaf is removed", () => {
+        const leafs = emptyLeafs();
+        updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Insert, key(1), smallValue(0xaa)]]);
+        const res = updateLeafs(leafs, blake2b, [[StateEntryUpdateAction.Remove, key(1), BytesBlob.empty()]]);
+        assert.deepStrictEqual(res.values, []);
+        assert.deepStrictEqual(res.removed, []);
+    });
+});

package/packages/jam/database/serialized-states-db.d.ts CHANGED Viewed

@@ -27,6 +27,7 @@ export declare class InMemorySerializedStates implements StatesDb<SerializedStat
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     close(): Promise<void>;
 }

package/packages/jam/database/serialized-states-db.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"serialized-states-db.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/serialized-states-db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE5E,qCAAqC;AACrC,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;AAEhG,sCAAsC;AACtC,qBAAa,wBAAyB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAc1G,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAd1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAyE;IAC5F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8D;WAE1E,GAAG,CAAC,EAAE,SAAS,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAA;KAAE;IAKxD,MAAM,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAIpF,OAAO;IAKD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAiBxG,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAIpE,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAsBxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAkB5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
1	+ {"version":3,"file":"serialized-states-db.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/serialized-states-db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE5E,qCAAqC;AACrC,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;AAEhG,sCAAsC;AACtC,qBAAa,wBAAyB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAc1G,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAd1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAyE;IAC5F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8D;WAE1E,GAAG,CAAC,EAAE,SAAS,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAA;KAAE;IAKxD,MAAM,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAIpF,OAAO;IAKD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAiBxG,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAIpE,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAsBxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAkB5D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAK7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}

package/packages/jam/database/serialized-states-db.js CHANGED Viewed

@@ -70,6 +70,10 @@ export class InMemorySerializedStates {
         });
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(_headers) {
+        // No value pruning here: this in-memory db keeps every value in a plain map
+        // and is not the long-running fuzz target the refcounting is meant for.
+    }
     markUnused(header) {
         this.db.delete(header);
     }

package/packages/jam/database/states.d.ts CHANGED Viewed

@@ -33,6 +33,16 @@ export interface StatesDb<T extends State = State> {
     updateAndSetState(header: HeaderHash, state: T, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     /** Retrieve posterior state of given header. */
     getState(header: HeaderHash): T | null;
+    /**
+     * Notify the backend about newly finalized blocks, in ancestor-first order.
+     *
+     * Backends that share data between states (e.g. a content-addressed values DB)
+     * use this to move the blocks' references from speculative to finalized.
+     * MUST be called before `markUnused` of the same finality round, otherwise
+     * the newly finalized blocks would be treated as dead forks and values still
+     * referenced by the finalized state could be dropped.
+     */
+    commitFinalized(headers: HeaderHash[]): void;
     /** Mark state as no longer needed. Backend may remove it asynchronously. */
     markUnused(header: HeaderHash): void;
     /**
@@ -56,6 +66,7 @@ export declare class InMemoryStates implements StatesDb<InMemoryState> {
     /** Insert a full state into the database. */
     insertInitialState(headerHash: HeaderHash, state: InMemoryState): Promise<Result<OK, StateUpdateError>>;
     getState(headerHash: HeaderHash): InMemoryState | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     close(): Promise<void>;
 }

package/packages/jam/database/states.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
1	+ {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC;;;;;;;;OAQG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE7C,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAI7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}

package/packages/jam/database/states.js CHANGED Viewed

@@ -53,6 +53,9 @@ export class InMemoryStates {
         }
         return InMemoryState.copyFrom(this.spec, state, state.intoServicesData());
     }
+    commitFinalized(_headers) {
+        // nothing to do: every state is a full, independent copy.
+    }
     markUnused(header) {
         this.db.delete(header);
     }

package/packages/jam/database/value-refs.d.ts ADDED Viewed

@@ -0,0 +1,117 @@
+import type { HeaderHash } from "#@typeberry/block";
+import type { ValueHash } from "#@typeberry/trie";
+/** The value hashes a single block introduced and stopped referencing. */
+export interface ValueDelta {
+    /** Value hashes (non-embedded) that the block started referencing. */
+    inserted: ValueHash[];
+    /** Value hashes (non-embedded) that the block stopped referencing. */
+    removed: ValueHash[];
+}
+/**
+ * Synchronous, read-only view of the persisted refcounting state.
+ *
+ * Both LMDB and fjall offer synchronous reads, so reads can go straight
+ * to the backing store. All writes go through `ValueRefsUpdate` batches
+ * instead, since persistent backends can only write asynchronously
+ * (LMDB transactions, fjall inserts + persist).
+ */
+export interface ValueRefsReader {
+    /** How many leaves of the finalized-tip state reference the value. Missing keys read as `0`. */
+    getFinalizedCount(hash: ValueHash): number;
+    /** How many surviving, not-yet-finalized blocks inserted the value. Missing keys read as `0`. */
+    getPendingCount(hash: ValueHash): number;
+    /** The value delta of a not-yet-finalized block, if known. */
+    getDelta(header: HeaderHash): ValueDelta | undefined;
+}
+/**
+ * A batch of refcounting mutations produced by a single `ValueRefs` operation.
+ *
+ * The backend is responsible for applying the batch using its own write
+ * primitive - ideally atomically with the state write that triggered it
+ * (one LMDB transaction, one fjall persist).
+ *
+ * Counts are absolute values rather than increments, so applying the same
+ * update more than once (e.g. on crash-replay) is harmless.
+ *
+ * When atomicity is not available, apply in field order and `removeValues`
+ * strictly last: a crash after counts are persisted but before values are
+ * removed only leaks values, while the opposite order could drop a value
+ * the persisted counts still consider referencd.
+ */
+export interface ValueRefsUpdate {
+    /** New absolute finalized counts. Count `0` means the entry should be removed. */
+    finalizedCounts: [ValueHash, number][];
+    /** New absolute pending counts. Count `0` means the entry should be removed. */
+    pendingCounts: [ValueHash, number][];
+    /** Deltas of freshly imported blocks to persist. */
+    setDeltas: [HeaderHash, ValueDelta][];
+    /** Deltas consumed by finalization or fork pruning. */
+    removeDeltas: HeaderHash[];
+    /** Values that are no longer referenced and can be removed from the values DB. */
+    removeValues: ValueHash[];
+}
+/** `true` if applying the update would not change anything. */
+export declare function isEmptyUpdate(update: ValueRefsUpdate): boolean;
+/**
+ * Decides when a content-addressed value can be removed from the values DB.
+ *
+ * A value is needed as long as some surviving state references it. Surviving
+ * states are the finalized tip plus its unfinalized descendants, so we track:
+ * - `finalized`: how many leaves in the current finalized-tip state reference V,
+ *   advanced strictly by replaying finalized blocks' deltas (never on prune);
+ * - `pending`: how many surviving, not-yet-finalized blocks inserted V (an
+ *   over-approximation of unfinalized references).
+ *
+ * A value is removable exactly when both counts reach zero.
+ *
+ * This class only makes decisions: every operation reads the current state
+ * through `ValueRefsReader` and returns a `ValueRefsUpdate` describing what
+ * should change. Nothing is written here - the owning backend applies the
+ * update with whatever consistency guarantees it can provide.
+ */
+export declare class ValueRefs {
+    private readonly reader;
+    constructor(reader: ValueRefsReader);
+    /** Record values referenced by the genesis / initial finalized state. */
+    onInitial(inserted: ValueHash[]): ValueRefsUpdate;
+    /**
+     * Record a freshly imported, not-yet-finalized block.
+     *
+     * Importing the same header twice is a no-op, otherwise the second import
+     * would double-count `pending` references and pin the values forever.
+     */
+    onImport(header: HeaderHash, delta: ValueDelta): ValueRefsUpdate;
+    /**
+     * Apply the value deltas of newly finalized blocks, in finalized (ancestor-first) order.
+     *
+     * Moves each inserted value from `pending` to `finalized`, drops `finalized`
+     * references for removed values, and collects anything that becomes unreferenced.
+     */
+    commitFinalized(headers: HeaderHash[]): ValueRefsUpdate;
+    /**
+     * Release the speculative references of a state being discarded.
+     *
+     * For a header that was still unfinalized (a dead fork) the returned update
+     * releases its inserted values, which may become collectable. For an already
+     * finalized state the update is empty (check with `isEmptyUpdate`): its delta
+     * was consumed on finality and its values are accounted for in `finalized`.
+     */
+    releaseUnfinalized(header: HeaderHash): ValueRefsUpdate;
+}
+/**
+ * In-memory refcounting store, reusable by the in-memory and hybrid states DBs
+ * (the hybrids cannot resume from disk anyway, so persisting counts buys nothing).
+ *
+ * NOTE: `apply` does not touch the values DB - the caller owns it and must
+ * handle `update.removeValues` itself.
+ */
+export declare class InMemoryValueRefsStore implements ValueRefsReader {
+    private readonly finalized;
+    private readonly pending;
+    private readonly deltas;
+    getFinalizedCount(hash: ValueHash): number;
+    getPendingCount(hash: ValueHash): number;
+    getDelta(header: HeaderHash): ValueDelta | undefined;
+    apply(update: ValueRefsUpdate): void;
+}
+//# sourceMappingURL=value-refs.d.ts.map

package/packages/jam/database/value-refs.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"value-refs.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/value-refs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,sEAAsE;IACtE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,sEAAsE;IACtE,OAAO,EAAE,SAAS,EAAE,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B,gGAAgG;IAChG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3C,iGAAiG;IACjG,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IACzC,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;CACtD;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,eAAe;IAC9B,kFAAkF;IAClF,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACvC,gFAAgF;IAChF,aAAa,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACrC,oDAAoD;IACpD,SAAS,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;IACtC,uDAAuD;IACvD,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,kFAAkF;IAClF,YAAY,EAAE,SAAS,EAAE,CAAC;CAC3B;AAED,+DAA+D;AAC/D,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAQ9D;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,eAAe;IAEpD,yEAAyE;IACzE,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,eAAe;IAQjD;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,eAAe;IAYhE;;;;;OAKG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,eAAe;IAoBvD;;;;;;;OAOG;IACH,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe;CAYxD;AA0ED;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,eAAe;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2D;IACrF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2D;IACnF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgE;IAEvF,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAI1C,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS;IAIpD,KAAK,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;CAUrC"}

package/packages/jam/database/value-refs.js ADDED Viewed

@@ -0,0 +1,206 @@
+import { HashDictionary } from "#@typeberry/collections";
+/** `true` if applying the update would not change anything. */
+export function isEmptyUpdate(update) {
+    return (update.finalizedCounts.length === 0 &&
+        update.pendingCounts.length === 0 &&
+        update.setDeltas.length === 0 &&
+        update.removeDeltas.length === 0 &&
+        update.removeValues.length === 0);
+}
+/**
+ * Decides when a content-addressed value can be removed from the values DB.
+ *
+ * A value is needed as long as some surviving state references it. Surviving
+ * states are the finalized tip plus its unfinalized descendants, so we track:
+ * - `finalized`: how many leaves in the current finalized-tip state reference V,
+ *   advanced strictly by replaying finalized blocks' deltas (never on prune);
+ * - `pending`: how many surviving, not-yet-finalized blocks inserted V (an
+ *   over-approximation of unfinalized references).
+ *
+ * A value is removable exactly when both counts reach zero.
+ *
+ * This class only makes decisions: every operation reads the current state
+ * through `ValueRefsReader` and returns a `ValueRefsUpdate` describing what
+ * should change. Nothing is written here - the owning backend applies the
+ * update with whatever consistency guarantees it can provide.
+ */
+export class ValueRefs {
+    reader;
+    constructor(reader) {
+        this.reader = reader;
+    }
+    /** Record values referenced by the genesis / initial finalized state. */
+    onInitial(inserted) {
+        const update = new UpdateBuilder(this.reader);
+        for (const v of inserted) {
+            update.incFinalized(v);
+        }
+        return update.build();
+    }
+    /**
+     * Record a freshly imported, not-yet-finalized block.
+     *
+     * Importing the same header twice is a no-op, otherwise the second import
+     * would double-count `pending` references and pin the values forever.
+     */
+    onImport(header, delta) {
+        const update = new UpdateBuilder(this.reader);
+        if (update.getDelta(header) !== undefined) {
+            return update.build();
+        }
+        update.setDelta(header, delta);
+        for (const v of delta.inserted) {
+            update.incPending(v);
+        }
+        return update.build();
+    }
+    /**
+     * Apply the value deltas of newly finalized blocks, in finalized (ancestor-first) order.
+     *
+     * Moves each inserted value from `pending` to `finalized`, drops `finalized`
+     * references for removed values, and collects anything that becomes unreferenced.
+     */
+    commitFinalized(headers) {
+        const update = new UpdateBuilder(this.reader);
+        for (const header of headers) {
+            const delta = update.getDelta(header);
+            if (delta === undefined) {
+                // already committed, or not tracked (e.g. genesis)
+                continue;
+            }
+            for (const v of delta.inserted) {
+                update.incFinalized(v);
+                update.decPending(v);
+            }
+            for (const v of delta.removed) {
+                update.decFinalized(v);
+            }
+            update.removeDelta(header);
+        }
+        return update.build();
+    }
+    /**
+     * Release the speculative references of a state being discarded.
+     *
+     * For a header that was still unfinalized (a dead fork) the returned update
+     * releases its inserted values, which may become collectable. For an already
+     * finalized state the update is empty (check with `isEmptyUpdate`): its delta
+     * was consumed on finality and its values are accounted for in `finalized`.
+     */
+    releaseUnfinalized(header) {
+        const update = new UpdateBuilder(this.reader);
+        const delta = update.getDelta(header);
+        if (delta === undefined) {
+            return update.build();
+        }
+        for (const v of delta.inserted) {
+            update.decPending(v);
+        }
+        update.removeDelta(header);
+        return update.build();
+    }
+}
+/**
+ * Accumulates mutations of a single operation as an overlay over the reader,
+ * so that later steps observe earlier ones (e.g. several blocks finalized at once).
+ */
+class UpdateBuilder {
+    reader;
+    finalized = HashDictionary.new();
+    pending = HashDictionary.new();
+    setDeltas = HashDictionary.new();
+    removedDeltas = HashDictionary.new();
+    /** Values that lost a reference and may need removal - verified against final counts in `build`. */
+    removalCandidates = HashDictionary.new();
+    constructor(reader) {
+        this.reader = reader;
+    }
+    getDelta(header) {
+        if (this.removedDeltas.has(header)) {
+            return undefined;
+        }
+        return this.setDeltas.get(header) ?? this.reader.getDelta(header);
+    }
+    setDelta(header, delta) {
+        this.setDeltas.set(header, delta);
+    }
+    removeDelta(header) {
+        this.removedDeltas.set(header, header);
+    }
+    incFinalized(v) {
+        this.finalized.set(v, this.getFinalized(v) + 1);
+    }
+    decFinalized(v) {
+        this.finalized.set(v, Math.max(0, this.getFinalized(v) - 1));
+        this.removalCandidates.set(v, v);
+    }
+    incPending(v) {
+        this.pending.set(v, this.getPending(v) + 1);
+    }
+    decPending(v) {
+        this.pending.set(v, Math.max(0, this.getPending(v) - 1));
+        this.removalCandidates.set(v, v);
+    }
+    build() {
+        const removeValues = [];
+        for (const v of this.removalCandidates.values()) {
+            if (this.getFinalized(v) === 0 && this.getPending(v) === 0) {
+                removeValues.push(v);
+            }
+        }
+        return {
+            finalizedCounts: Array.from(this.finalized),
+            pendingCounts: Array.from(this.pending),
+            setDeltas: Array.from(this.setDeltas),
+            removeDeltas: Array.from(this.removedDeltas.keys()),
+            removeValues,
+        };
+    }
+    getFinalized(v) {
+        return this.finalized.get(v) ?? this.reader.getFinalizedCount(v);
+    }
+    getPending(v) {
+        return this.pending.get(v) ?? this.reader.getPendingCount(v);
+    }
+}
+/**
+ * In-memory refcounting store, reusable by the in-memory and hybrid states DBs
+ * (the hybrids cannot resume from disk anyway, so persisting counts buys nothing).
+ *
+ * NOTE: `apply` does not touch the values DB - the caller owns it and must
+ * handle `update.removeValues` itself.
+ */
+export class InMemoryValueRefsStore {
+    finalized = HashDictionary.new();
+    pending = HashDictionary.new();
+    deltas = HashDictionary.new();
+    getFinalizedCount(hash) {
+        return this.finalized.get(hash) ?? 0;
+    }
+    getPendingCount(hash) {
+        return this.pending.get(hash) ?? 0;
+    }
+    getDelta(header) {
+        return this.deltas.get(header);
+    }
+    apply(update) {
+        applyCounts(this.finalized, update.finalizedCounts);
+        applyCounts(this.pending, update.pendingCounts);
+        for (const [header, delta] of update.setDeltas) {
+            this.deltas.set(header, delta);
+        }
+        for (const header of update.removeDeltas) {
+            this.deltas.delete(header);
+        }
+    }
+}
+function applyCounts(store, counts) {
+    for (const [hash, count] of counts) {
+        if (count === 0) {
+            store.delete(hash);
+        }
+        else {
+            store.set(hash, count);
+        }
+    }
+}

package/packages/jam/database/value-refs.test.d.ts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=value-refs.test.d.ts.map

package/packages/jam/database/value-refs.test.d.ts.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"value-refs.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/value-refs.test.ts"],"names":[],"mappings":""}

package/packages/jam/database/value-refs.test.js ADDED Viewed

@@ -0,0 +1,165 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { Bytes } from "#@typeberry/bytes";
+import { HASH_SIZE } from "#@typeberry/hash";
+import { InMemoryValueRefsStore, isEmptyUpdate, ValueRefs, } from "./value-refs.js";
+function vh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+/** Tracks which value hashes are present in the (simulated) values DB. */
+class TrackingValues {
+    present = new Set();
+    write(h) {
+        this.present.add(h.toString());
+    }
+    delete(h) {
+        this.present.delete(h.toString());
+    }
+    has(h) {
+        return this.present.has(h.toString());
+    }
+}
+function setup() {
+    const store = new InMemoryValueRefsStore();
+    const refs = new ValueRefs(store);
+    const values = new TrackingValues();
+    // what every backend is expected to do with an update
+    const apply = (update) => {
+        store.apply(update);
+        for (const v of update.removeValues) {
+            values.delete(v);
+        }
+    };
+    return { refs, store, values, apply };
+}
+/** Simulate importing a block: write inserted values + record the delta. */
+function importBlock({ refs, values, apply }, header, delta) {
+    const full = { inserted: delta.inserted ?? [], removed: delta.removed ?? [] };
+    for (const v of full.inserted) {
+        values.write(v);
+    }
+    apply(refs.onImport(header, full));
+}
+describe("ValueRefs", () => {
+    it("deletes a value once its only finalized reference is removed", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1)]));
+        assert.ok(values.has(V), "kept while referenced by finalized tip");
+        importBlock(s, hh(2), { removed: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        assert.ok(!values.has(V), "deleted after last finalized reference removed");
+    });
+    it("keeps a value referenced under multiple keys until all references are gone", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1)]));
+        importBlock(s, hh(2), { inserted: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        importBlock(s, hh(3), { removed: [V] });
+        apply(refs.commitFinalized([hh(3)]));
+        assert.ok(values.has(V), "kept while one reference remains");
+        importBlock(s, hh(4), { removed: [V] });
+        apply(refs.commitFinalized([hh(4)]));
+        assert.ok(!values.has(V), "deleted after the last reference");
+    });
+    it("keeps a value removed on the finalized chain while an unfinalized fork re-adds it", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        // genesis already references V
+        apply(refs.onInitial([V]));
+        values.write(V);
+        // finalized chain removes V
+        importBlock(s, hh(10), { removed: [V] });
+        // an unfinalized fork re-adds V
+        importBlock(s, hh(20), { inserted: [V] });
+        apply(refs.commitFinalized([hh(10)]));
+        assert.ok(values.has(V), "kept because an unfinalized fork still references V");
+    });
+    it("collects a value that lived only on a discarded fork", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const W = vh(2);
+        importBlock(s, hh(30), { inserted: [W] });
+        assert.ok(values.has(W));
+        const update = refs.releaseUnfinalized(hh(30));
+        apply(update);
+        assert.strictEqual(isEmptyUpdate(update), false);
+        assert.ok(!values.has(W), "dead-fork-only value is collected");
+    });
+    it("does not touch finalized references when releasing an already finalized header", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1)]));
+        const update = refs.releaseUnfinalized(hh(1));
+        apply(update);
+        assert.strictEqual(isEmptyUpdate(update), true, "header was already finalized");
+        assert.ok(values.has(V), "finalized value is untouched");
+    });
+    it("ignores unknown headers in commitFinalized", () => {
+        const { refs } = setup();
+        const update = refs.commitFinalized([hh(99)]);
+        assert.strictEqual(isEmptyUpdate(update), true);
+    });
+    it("ignores a duplicate import of the same header", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        const second = refs.onImport(hh(1), { inserted: [V], removed: [] });
+        assert.strictEqual(isEmptyUpdate(second), true, "duplicate import is a no-op");
+        // had the duplicate bumped `pending` twice, releasing the fork would leave
+        // a dangling count and the value would never be collected
+        apply(refs.releaseUnfinalized(hh(1)));
+        assert.ok(!values.has(V));
+    });
+    it("counts a header only once within a single commitFinalized call", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1), hh(1)]));
+        importBlock(s, hh(2), { removed: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        assert.ok(!values.has(V), "a double-counted finalized reference would keep the value alive");
+    });
+    it("does not remove a value re-referenced later within the same batch", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        apply(refs.onInitial([V]));
+        values.write(V);
+        // one block removes V, its descendant re-adds it; both finalize at once
+        importBlock(s, hh(1), { removed: [V] });
+        importBlock(s, hh(2), { inserted: [V] });
+        const update = refs.commitFinalized([hh(1), hh(2)]);
+        apply(update);
+        assert.strictEqual(update.removeValues.length, 0);
+        assert.ok(values.has(V), "the re-added reference keeps the value alive");
+    });
+    it("emits absolute counts so re-applying an update is harmless", () => {
+        const s = setup();
+        const { refs, store, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        const update = refs.commitFinalized([hh(1)]);
+        apply(update);
+        // crash-replay: same batch applied again
+        apply(update);
+        assert.strictEqual(store.getFinalizedCount(V), 1);
+        assert.strictEqual(store.getPendingCount(V), 0);
+        importBlock(s, hh(2), { removed: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        assert.ok(!values.has(V), "refcounting still exact after replay");
+    });
+});

package/packages/jam/database-fjall/hybrid-states.d.ts CHANGED Viewed

@@ -39,6 +39,14 @@ export declare class FjallValuesSession {
  *
  * Construction is async, and value writes are flushed explicitly, because fjall
  * has no transaction primitive.
+ *
+ * Values that no longer belong to any surviving state are removed from fjall,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous run
+ * are never collected. An instance backed by a shared session (fuzz reset
+ * reuse) only ever prunes values it inserted itself, since its refcounts start
+ * empty - values left behind by earlier resets stay untouched.
  */
 export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
     private readonly spec;
@@ -66,13 +74,19 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     private readonly valuesDb;
     /** Shared content-addressed values partition (owned by `session`). */
     private readonly values;
+    private readonly refsStore;
+    private readonly refs;
+    private pendingCleanup;
     private constructor();
     insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     diskSizeInBytes(): number | null;
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    private applyRefs;
     close(): Promise<void>;
     /** Write new large values to fjall in a single batch, then flush. */
     private writeValues;

package/packages/jam/database-fjall/hybrid-states.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,~~EACjB~~,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,~~EAGjB~~,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAa,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAgB,MAAM,WAAW,CAAC;AAI3F;;;;;;;;;GASG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,SAAS;IAH5B,OAAO;IAMP,4EAA4E;WAC/D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAM9F,mEAAmE;IAC7D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,wDAAwD;IACxD,WAAW,IAAI,MAAM,GAAG,IAAI;IAI5B,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED~~;;;;;;;;;GASG~~;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;~~IAsCxG~~,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,WAAW;~~WAzCjB~~,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,cAAc,GACf,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC;;;;;;;OAOG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,GAAG,sBAAsB;IAI1G,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;~~IAEnC~~,OAAO;IAWD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;~~IAcxG~~,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;~~IAkBlC~~,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;~~IAMpC~~,eAAe,IAAI,MAAM,GAAG,IAAI;~~IAI1B~~,KAAK;~~IAQX~~,qEAAqE;YACvD,WAAW;~~IAezB~~,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}
1	+ {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAa,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAgB,MAAM,WAAW,CAAC;AAI3F;;;;;;;;;GASG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,SAAS;IAH5B,OAAO;IAMP,4EAA4E;WAC/D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAM9F,mEAAmE;IAC7D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,wDAAwD;IACxD,WAAW,IAAI,MAAM,GAAG,IAAI;IAI5B,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0CxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,WAAW;WA7CjB,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,cAAc,GACf,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC;;;;;;;OAOG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,GAAG,sBAAsB;IAI1G,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,OAAO;IAWD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAIhC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAcX,KAAK;IAWX,qEAAqE;YACvD,WAAW;IAoBzB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-fjall/hybrid-states.js CHANGED Viewed

@@ -1,5 +1,5 @@
 import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
 import { Logger } from "#@typeberry/logger";
 import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
 import { leafComparator } from "#@typeberry/trie";
@@ -53,6 +53,14 @@ export class FjallValuesSession {
  *
  * Construction is async, and value writes are flushed explicitly, because fjall
  * has no transaction primitive.
+ *
+ * Values that no longer belong to any surviving state are removed from fjall,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous run
+ * are never collected. An instance backed by a shared session (fuzz reset
+ * reuse) only ever prunes values it inserted itself, since its refcounts start
+ * empty - values left behind by earlier resets stay untouched.
  */
 export class HybridSerializedStates {
     spec;
@@ -80,6 +88,10 @@ export class HybridSerializedStates {
     valuesDb;
     /** Shared content-addressed values partition (owned by `session`). */
     values;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
     constructor(spec, blake2b, session,
     /** Whether `close()` should close the underlying session. */
     ownsSession) {
@@ -97,13 +109,14 @@ export class HybridSerializedStates {
             return res;
         }
         this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async updateAndSetState(header, state, update) {
         const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
         // Clone the leaf set before mutating: the previous state keeps using its own.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
         const res = await this.writeValues(values);
         if (res.isError) {
             // Leave the caller's state untouched: its new leaves would reference
@@ -114,6 +127,7 @@ export class HybridSerializedStates {
         // values are durably written.
         state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
         this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -127,15 +141,36 @@ export class HybridSerializedStates {
         const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
-        // We only remove the state from memory - values are not pruned at all,
-        // but since they are stored on disk we should be safe.
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.inMemStates.delete(header);
     }
     diskSizeInBytes() {
         return this.session.sizeInBytes();
     }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value. No explicit
+        // persist - removals are flushed together with the next value write.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => Promise.all(update.removeValues.map((v) => this.values.remove(v.raw))))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
+    }
     async close() {
+        // Finish any queued value removals before tearing down (or releasing) the
+        // session - they operate on the shared `values` partition.
+        await this.pendingCleanup;
         // Instances backed by a shared session (fuzz reset reuse) keep the keyspace
         // open for the next reset. The session owner closes it once.
         if (this.ownsSession) {
@@ -148,6 +183,11 @@ export class HybridSerializedStates {
             return Result.ok(OK);
         }
         try {
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
             const entries = values.map(([hash, val]) => ({ key: hash.raw, value: val.raw }));
             await this.values.insertBatch(entries);
             await this.session.persist();

package/packages/jam/database-lmdb/hybrid-states.d.ts CHANGED Viewed

@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
  * Reads go straight to lmdb, which keeps its own page cache.
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
  * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
  */
 export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
     private readonly spec;
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     private readonly inMemStates;
     private readonly lmdbValues;
     private readonly valuesDb;
+    private readonly refsStore;
+    private readonly refs;
+    private pendingCleanup;
     static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
         spec: ChainSpec;
         blake2b: Blake2b;
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    private applyRefs;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;
     /** Write new large values to LMDB in one transaction. */

package/packages/jam/database-lmdb/hybrid-states.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,~~EACjB~~,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,~~EAGjB~~,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;~~AAC~~/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;~~AAG9C;;;;;;;GAOG~~;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;~~IA0BxG~~,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;~~IA3BvB~~,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;~~IAEpC~~,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;~~IAcxG~~,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;~~IAkBlC~~,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;~~IAMpC~~,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;~~IAKX~~,yDAAyD;YAC3C,WAAW;~~IAgBzB~~,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
1	+ {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-lmdb/hybrid-states.js CHANGED Viewed

@@ -1,9 +1,11 @@
 import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
+import { Logger } from "#@typeberry/logger";
 import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
 import { leafComparator } from "#@typeberry/trie";
 import { OK, Result } from "#@typeberry/utils";
 import { LmdbRoot } from "./root.js";
+const logger = Logger.new(import.meta.filename, "db");
 /**
  * Hybrid serialized-states db.
  *
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
  * Reads go straight to lmdb, which keeps its own page cache.
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
  * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
  */
 export class HybridSerializedStates {
     spec;
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
     lmdbValues;
     // A single shared values accessor reused by every `LeafDb` we hand out.
     valuesDb;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
     static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
         const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
         return new HybridSerializedStates(spec, blake2b, root);
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
             return res;
         }
         this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async updateAndSetState(header, state, update) {
         const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
         // Clone the leaf set before mutating: the previous state keeps using its own.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
         const res = await this.writeValues(values);
         if (res.isError) {
             // Leave the caller's state untouched: its new leaves would reference
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
         // values are durably written.
         state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
         this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
         const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
-        // We only remove the state from memory - values are not pruned at all,
-        // but since they are stored on disk we should be safe.
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.inMemStates.delete(header);
     }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => this.lmdbValues.transaction(() => {
+            for (const v of update.removeValues) {
+                this.lmdbValues.remove(v.raw);
+            }
+        }))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
+    }
     diskSizeInBytes() {
         return this.root.sizeInBytes();
     }
     async close() {
+        await this.pendingCleanup;
         await this.lmdbValues.close();
         await this.root.close();
     }
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
             return Result.ok(OK);
         }
         try {
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
             await this.lmdbValues.transaction(() => {
                 for (const [hash, val] of values) {
                     this.lmdbValues.put(hash.raw, val.raw);

package/packages/jam/database-lmdb/states.d.ts CHANGED Viewed

@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
     updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(root: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;

package/packages/jam/database-lmdb/states.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
1	+ {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}

package/packages/jam/database-lmdb/states.js CHANGED Viewed

@@ -128,6 +128,11 @@ export class LmdbStates {
         }
         return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
     }
+    commitFinalized(_headers) {
+        // Values are never pruned here. This db survives restarts, so refcounting
+        // would need counts persisted (and crash-consistent) alongside the values.
+        // See the TODO above - not implemented until actually needed.
+    }
     markUnused(header) {
         this.states.removeSync(header.raw);
     }