@typeberry/lib 0.9.0-959889e → 0.9.0-a723180

package/packages/jam/database/value-refs.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=value-refs.test.d.ts.map

package/packages/jam/database/value-refs.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"value-refs.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/value-refs.test.ts"],"names":[],"mappings":""}

package/packages/jam/database/value-refs.test.js

@@ -0,0 +1,165 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { Bytes } from "#@typeberry/bytes";
+import { HASH_SIZE } from "#@typeberry/hash";
+import { InMemoryValueRefsStore, isEmptyUpdate, ValueRefs, } from "./value-refs.js";
+function vh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+/** Tracks which value hashes are present in the (simulated) values DB. */
+class TrackingValues {
+    present = new Set();
+    write(h) {
+        this.present.add(h.toString());
+    }
+    delete(h) {
+        this.present.delete(h.toString());
+    }
+    has(h) {
+        return this.present.has(h.toString());
+    }
+}
+function setup() {
+    const store = new InMemoryValueRefsStore();
+    const refs = new ValueRefs(store);
+    const values = new TrackingValues();
+    // what every backend is expected to do with an update
+    const apply = (update) => {
+        store.apply(update);
+        for (const v of update.removeValues) {
+            values.delete(v);
+        }
+    };
+    return { refs, store, values, apply };
+}
+/** Simulate importing a block: write inserted values + record the delta. */
+function importBlock({ refs, values, apply }, header, delta) {
+    const full = { inserted: delta.inserted ?? [], removed: delta.removed ?? [] };
+    for (const v of full.inserted) {
+        values.write(v);
+    }
+    apply(refs.onImport(header, full));
+}
+describe("ValueRefs", () => {
+    it("deletes a value once its only finalized reference is removed", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1)]));
+        assert.ok(values.has(V), "kept while referenced by finalized tip");
+        importBlock(s, hh(2), { removed: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        assert.ok(!values.has(V), "deleted after last finalized reference removed");
+    });
+    it("keeps a value referenced under multiple keys until all references are gone", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1)]));
+        importBlock(s, hh(2), { inserted: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        importBlock(s, hh(3), { removed: [V] });
+        apply(refs.commitFinalized([hh(3)]));
+        assert.ok(values.has(V), "kept while one reference remains");
+        importBlock(s, hh(4), { removed: [V] });
+        apply(refs.commitFinalized([hh(4)]));
+        assert.ok(!values.has(V), "deleted after the last reference");
+    });
+    it("keeps a value removed on the finalized chain while an unfinalized fork re-adds it", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        // genesis already references V
+        apply(refs.onInitial([V]));
+        values.write(V);
+        // finalized chain removes V
+        importBlock(s, hh(10), { removed: [V] });
+        // an unfinalized fork re-adds V
+        importBlock(s, hh(20), { inserted: [V] });
+        apply(refs.commitFinalized([hh(10)]));
+        assert.ok(values.has(V), "kept because an unfinalized fork still references V");
+    });
+    it("collects a value that lived only on a discarded fork", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const W = vh(2);
+        importBlock(s, hh(30), { inserted: [W] });
+        assert.ok(values.has(W));
+        const update = refs.releaseUnfinalized(hh(30));
+        apply(update);
+        assert.strictEqual(isEmptyUpdate(update), false);
+        assert.ok(!values.has(W), "dead-fork-only value is collected");
+    });
+    it("does not touch finalized references when releasing an already finalized header", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1)]));
+        const update = refs.releaseUnfinalized(hh(1));
+        apply(update);
+        assert.strictEqual(isEmptyUpdate(update), true, "header was already finalized");
+        assert.ok(values.has(V), "finalized value is untouched");
+    });
+    it("ignores unknown headers in commitFinalized", () => {
+        const { refs } = setup();
+        const update = refs.commitFinalized([hh(99)]);
+        assert.strictEqual(isEmptyUpdate(update), true);
+    });
+    it("ignores a duplicate import of the same header", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        const second = refs.onImport(hh(1), { inserted: [V], removed: [] });
+        assert.strictEqual(isEmptyUpdate(second), true, "duplicate import is a no-op");
+        // had the duplicate bumped `pending` twice, releasing the fork would leave
+        // a dangling count and the value would never be collected
+        apply(refs.releaseUnfinalized(hh(1)));
+        assert.ok(!values.has(V));
+    });
+    it("counts a header only once within a single commitFinalized call", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        apply(refs.commitFinalized([hh(1), hh(1)]));
+        importBlock(s, hh(2), { removed: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        assert.ok(!values.has(V), "a double-counted finalized reference would keep the value alive");
+    });
+    it("does not remove a value re-referenced later within the same batch", () => {
+        const s = setup();
+        const { refs, values, apply } = s;
+        const V = vh(1);
+        apply(refs.onInitial([V]));
+        values.write(V);
+        // one block removes V, its descendant re-adds it; both finalize at once
+        importBlock(s, hh(1), { removed: [V] });
+        importBlock(s, hh(2), { inserted: [V] });
+        const update = refs.commitFinalized([hh(1), hh(2)]);
+        apply(update);
+        assert.strictEqual(update.removeValues.length, 0);
+        assert.ok(values.has(V), "the re-added reference keeps the value alive");
+    });
+    it("emits absolute counts so re-applying an update is harmless", () => {
+        const s = setup();
+        const { refs, store, values, apply } = s;
+        const V = vh(1);
+        importBlock(s, hh(1), { inserted: [V] });
+        const update = refs.commitFinalized([hh(1)]);
+        apply(update);
+        // crash-replay: same batch applied again
+        apply(update);
+        assert.strictEqual(store.getFinalizedCount(V), 1);
+        assert.strictEqual(store.getPendingCount(V), 0);
+        importBlock(s, hh(2), { removed: [V] });
+        apply(refs.commitFinalized([hh(2)]));
+        assert.ok(!values.has(V), "refcounting still exact after replay");
+    });
+});

package/packages/jam/database-fjall/blocks.d.ts

@@ -0,0 +1,25 @@
+import { type BlockView, type ExtrinsicView, type HeaderHash, type HeaderView, type StateRootHash } from "#@typeberry/block";
+import type { ChainSpec } from "#@typeberry/config";
+import type { BlocksDb } from "#@typeberry/database/blocks.js";
+import { type WithHash } from "#@typeberry/hash";
+import { type FjallRoot } from "./root.js";
+/** fjall-backed block storage. */
+export declare class FjallBlocks implements BlocksDb {
+    private readonly chainSpec;
+    private readonly root;
+    private readonly headers;
+    private readonly extrinsics;
+    private readonly postStateRoots;
+    static open(chainSpec: ChainSpec, root: FjallRoot): Promise<FjallBlocks>;
+    private constructor();
+    setPostStateRoot(hash: HeaderHash, postStateRoot: StateRootHash): Promise<void>;
+    getPostStateRoot(hash: HeaderHash): StateRootHash | null;
+    insertBlock(block: WithHash<HeaderHash, BlockView>): Promise<void>;
+    setBestHeaderHash(hash: HeaderHash): Promise<void>;
+    getBestHeaderHash(): HeaderHash;
+    getHeader(hash: HeaderHash): HeaderView | null;
+    getExtrinsic(hash: HeaderHash): ExtrinsicView | null;
+    markUnused(hash: HeaderHash): void;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=blocks.d.ts.map

package/packages/jam/database-fjall/blocks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAK9F,kCAAkC;AAClC,qBAAa,WAAY,YAAW,QAAQ;IAWxC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc;WAdpB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAS9E,OAAO;IAQD,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrF,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQlD,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlE,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxD,iBAAiB,IAAI,UAAU;IAQ/B,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAQ9C,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQpD,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAQ5B,KAAK;CACZ"}

package/packages/jam/database-fjall/blocks.js

@@ -0,0 +1,86 @@
+import { Extrinsic, Header, } from "#@typeberry/block";
+import { Bytes } from "#@typeberry/bytes";
+import { Decoder } from "#@typeberry/codec";
+import { HASH_SIZE } from "#@typeberry/hash";
+import { Logger } from "#@typeberry/logger";
+import { toUint8Array } from "./root.js";
+const BEST_BLOCK = new TextEncoder().encode("best hash and posterior state root");
+const logger = Logger.new(import.meta.filename, "db");
+/** fjall-backed block storage. */
+export class FjallBlocks {
+    chainSpec;
+    root;
+    headers;
+    extrinsics;
+    postStateRoots;
+    static async open(chainSpec, root) {
+        const [headers, extrinsics, postStateRoots] = await Promise.all([
+            root.partition("headers"),
+            root.partition("extrinsics"),
+            root.partition("postStateRoots"),
+        ]);
+        return new FjallBlocks(chainSpec, root, headers, extrinsics, postStateRoots);
+    }
+    constructor(chainSpec, root, headers, extrinsics, postStateRoots) {
+        this.chainSpec = chainSpec;
+        this.root = root;
+        this.headers = headers;
+        this.extrinsics = extrinsics;
+        this.postStateRoots = postStateRoots;
+    }
+    async setPostStateRoot(hash, postStateRoot) {
+        await writable(this.postStateRoots, this.root).insert(hash.raw, postStateRoot.raw);
+    }
+    getPostStateRoot(hash) {
+        const postStateRoot = toUint8Array(this.postStateRoots.get(hash.raw));
+        if (postStateRoot === null) {
+            return null;
+        }
+        return Bytes.fromBlob(postStateRoot, HASH_SIZE).asOpaque();
+    }
+    async insertBlock(block) {
+        await Promise.all([
+            writable(this.headers, this.root).insert(block.hash.raw, block.data.header.view().encoded().raw),
+            writable(this.extrinsics, this.root).insert(block.hash.raw, block.data.extrinsic.view().encoded().raw),
+        ]);
+    }
+    async setBestHeaderHash(hash) {
+        await writable(this.headers, this.root).insert(BEST_BLOCK, hash.raw);
+        await this.root.persist();
+    }
+    getBestHeaderHash() {
+        const bestHeaderHash = toUint8Array(this.headers.get(BEST_BLOCK));
+        if (bestHeaderHash === null) {
+            return Bytes.zero(HASH_SIZE).asOpaque();
+        }
+        return Bytes.fromBlob(bestHeaderHash, HASH_SIZE).asOpaque();
+    }
+    getHeader(hash) {
+        const data = toUint8Array(this.headers.get(hash.raw));
+        if (data === null) {
+            return null;
+        }
+        return Decoder.decodeObject(Header.Codec.View, data, this.chainSpec);
+    }
+    getExtrinsic(hash) {
+        const data = toUint8Array(this.extrinsics.get(hash.raw));
+        if (data === null) {
+            return null;
+        }
+        return Decoder.decodeObject(Extrinsic.Codec.View, data, this.chainSpec);
+    }
+    markUnused(hash) {
+        void Promise.all([
+            writable(this.headers, this.root).remove(hash.raw),
+            writable(this.extrinsics, this.root).remove(hash.raw),
+            writable(this.postStateRoots, this.root).remove(hash.raw),
+        ]).catch((e) => logger.warn `Failed to prune block ${hash}: ${e}`);
+    }
+    async close() { }
+}
+function writable(partition, root) {
+    if (root.readOnly) {
+        throw new Error("Cannot write through a read-only fjall partition.");
+    }
+    return partition;
+}

package/packages/jam/database-fjall/blocks.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=blocks.test.d.ts.map

package/packages/jam/database-fjall/blocks.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"blocks.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/blocks.test.ts"],"names":[],"mappings":""}

package/packages/jam/database-fjall/blocks.test.js

@@ -0,0 +1,62 @@
+import assert from "node:assert";
+import * as fs from "node:fs";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { testBlockView } from "#@typeberry/block/test-helpers.js";
+import { Bytes } from "#@typeberry/bytes";
+import { tinyChainSpec } from "#@typeberry/config";
+import { Blake2b, HASH_SIZE, WithHash } from "#@typeberry/hash";
+import { FjallBlocks } from "./blocks.js";
+import { FjallRoot } from "./root.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+describe("Fjall blocks database", () => {
+    let tmpDir = "";
+    beforeEach(() => {
+        tmpDir = fs.mkdtempSync("typeberry-fjall-blocks-");
+    });
+    afterEach(() => {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+    it("sets and retrieves the best header hash", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const blocks = await FjallBlocks.open(tinyChainSpec, root);
+        try {
+            await blocks.setBestHeaderHash(Bytes.fill(HASH_SIZE, 5).asOpaque());
+            assert.strictEqual(blocks.getBestHeaderHash().toString(), "0x0505050505050505050505050505050505050505050505050505050505050505");
+        }
+        finally {
+            await blocks.close();
+            await root.close();
+        }
+    });
+    it("sets and retrieves post state roots", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const blocks = await FjallBlocks.open(tinyChainSpec, root);
+        try {
+            await blocks.setPostStateRoot(Bytes.fill(HASH_SIZE, 5).asOpaque(), Bytes.fill(HASH_SIZE, 10).asOpaque());
+            assert.strictEqual(blocks.getPostStateRoot(Bytes.fill(HASH_SIZE, 1).asOpaque())?.toString(), undefined);
+            assert.strictEqual(blocks.getPostStateRoot(Bytes.fill(HASH_SIZE, 5).asOpaque())?.toString(), "0x0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a");
+        }
+        finally {
+            await blocks.close();
+            await root.close();
+        }
+    });
+    it("stores and retrieves a block", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const blocks = await FjallBlocks.open(tinyChainSpec, root);
+        try {
+            const block = testBlockView();
+            const headerHash = blake2b.hashBytes(block.header.view().encoded()).asOpaque();
+            await blocks.insertBlock(WithHash.new(headerHash, block));
+            assert.deepStrictEqual(blocks.getHeader(headerHash)?.materialize(), block.header.materialize());
+            assert.deepStrictEqual(blocks.getExtrinsic(headerHash)?.materialize(), block.extrinsic.materialize());
+        }
+        finally {
+            await blocks.close();
+            await root.close();
+        }
+    });
+});

package/packages/jam/database-fjall/hybrid-states.d.ts

@@ -39,6 +39,14 @@ export declare class FjallValuesSession {
  *
  * Construction is async, and value writes are flushed explicitly, because fjall
  * has no transaction primitive.
+ *
+ * Values that no longer belong to any surviving state are removed from fjall,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous run
+ * are never collected. An instance backed by a shared session (fuzz reset
+ * reuse) only ever prunes values it inserted itself, since its refcounts start
+ * empty - values left behind by earlier resets stay untouched.
  */
 export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
     private readonly spec;
@@ -66,13 +74,19 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     private readonly valuesDb;
     /** Shared content-addressed values partition (owned by `session`). */
     private readonly values;
+    private readonly refsStore;
+    private readonly refs;
+    private pendingCleanup;
     private constructor();
     insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     diskSizeInBytes(): number | null;
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    private applyRefs;
     close(): Promise<void>;
     /** Write new large values to fjall in a single batch, then flush. */
     private writeValues;

package/packages/jam/database-fjall/hybrid-states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAa,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAgB,MAAM,WAAW,CAAC;AAI3F;;;;;;;;;GASG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,SAAS;IAH5B,OAAO;IAMP,4EAA4E;WAC/D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAM9F,mEAAmE;IAC7D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,wDAAwD;IACxD,WAAW,IAAI,MAAM,GAAG,IAAI;IAI5B,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;;;;;GASG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAsCxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,WAAW;WAzCjB,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,cAAc,GACf,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC;;;;;;;OAOG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,GAAG,sBAAsB;IAI1G,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IAEnC,OAAO;IAWD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAQX,qEAAqE;YACvD,WAAW;IAezB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}
+{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAa,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAgB,MAAM,WAAW,CAAC;AAI3F;;;;;;;;;GASG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,SAAS;IAH5B,OAAO;IAMP,4EAA4E;WAC/D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAc9F,mEAAmE;IAC7D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,wDAAwD;IACxD,WAAW,IAAI,MAAM,GAAG,IAAI;IAI5B,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0CxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,WAAW;WA7CjB,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,cAAc,GACf,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC;;;;;;;OAOG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,GAAG,sBAAsB;IAI1G,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,OAAO;IAWD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAIhC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAcX,KAAK;IAWX,qEAAqE;YACvD,WAAW;IAoBzB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-fjall/hybrid-states.js

@@ -1,5 +1,5 @@
 import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
 import { Logger } from "#@typeberry/logger";
 import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
 import { leafComparator } from "#@typeberry/trie";
@@ -27,9 +27,18 @@ export class FjallValuesSession {
     }
     /** Open (or create) the keyspace at `dbPath` and its `values` partition. */
     static async open(dbPath, options = {}) {
+        if (options.readOnly === true) {
+            throw new Error("FjallValuesSession requires a writable keyspace.");
+        }
         const root = await FjallRoot.open(dbPath, options);
-        const values = await root.partition("values");
-        return new FjallValuesSession(root, values);
+        try {
+            const values = await root.writablePartition("values");
+            return new FjallValuesSession(root, values);
+        }
+        catch (e) {
+            await root.close();
+            throw e;
+        }
     }
     /** Flush the journal to disk (a no-op for ephemeral keyspaces). */
     async persist() {
@@ -53,6 +62,14 @@ export class FjallValuesSession {
  *
  * Construction is async, and value writes are flushed explicitly, because fjall
  * has no transaction primitive.
+ *
+ * Values that no longer belong to any surviving state are removed from fjall,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous run
+ * are never collected. An instance backed by a shared session (fuzz reset
+ * reuse) only ever prunes values it inserted itself, since its refcounts start
+ * empty - values left behind by earlier resets stay untouched.
  */
 export class HybridSerializedStates {
     spec;
@@ -80,6 +97,10 @@ export class HybridSerializedStates {
     valuesDb;
     /** Shared content-addressed values partition (owned by `session`). */
     values;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
     constructor(spec, blake2b, session, 
     /** Whether `close()` should close the underlying session. */
     ownsSession) {
@@ -97,13 +118,14 @@ export class HybridSerializedStates {
             return res;
         }
         this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async updateAndSetState(header, state, update) {
         const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
         // Clone the leaf set before mutating: the previous state keeps using its own.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
         const res = await this.writeValues(values);
         if (res.isError) {
             // Leave the caller's state untouched: its new leaves would reference
@@ -114,6 +136,7 @@ export class HybridSerializedStates {
         // values are durably written.
         state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
         this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -127,15 +150,36 @@ export class HybridSerializedStates {
         const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
-        // We only remove the state from memory - values are not pruned at all,
-        // but since they are stored on disk we should be safe.
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.inMemStates.delete(header);
     }
     diskSizeInBytes() {
         return this.session.sizeInBytes();
     }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value. No explicit
+        // persist - removals are flushed together with the next value write.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => Promise.all(update.removeValues.map((v) => this.values.remove(v.raw))))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
+    }
     async close() {
+        // Finish any queued value removals before tearing down (or releasing) the
+        // session - they operate on the shared `values` partition.
+        await this.pendingCleanup;
         // Instances backed by a shared session (fuzz reset reuse) keep the keyspace
         // open for the next reset. The session owner closes it once.
         if (this.ownsSession) {
@@ -148,6 +192,11 @@ export class HybridSerializedStates {
             return Result.ok(OK);
         }
         try {
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
             const entries = values.map(([hash, val]) => ({ key: hash.raw, value: val.raw }));
             await this.values.insertBatch(entries);
             await this.session.persist();

package/packages/jam/database-fjall/hybrid-states.test.js

@@ -1,12 +1,14 @@
 import assert from "node:assert";
 import * as fs from "node:fs";
 import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { tryAsServiceId } from "#@typeberry/block";
 import { Bytes, BytesBlob } from "#@typeberry/bytes";
 import { tinyChainSpec } from "#@typeberry/config";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
-import { InMemoryState } from "#@typeberry/state";
+import { InMemoryState, StorageItem, UpdateStorage, } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
 import { StateEntries } from "#@typeberry/state-merkleization";
-import { deepEqual, OK, Result } from "#@typeberry/utils";
+import { asOpaqueType, deepEqual, OK, Result } from "#@typeberry/utils";
 import { FjallValuesSession, HybridSerializedStates } from "./hybrid-states.js";
 let blake2b;
 before(async () => {
@@ -111,3 +113,93 @@ describe("Fjall hybrid serialized states", () => {
         }
     });
 });
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+const storageKey = asOpaqueType(BytesBlob.blobFromString("test-key"));
+/** A state update writing a single large (non-embedded) value under `storageKey`. */
+function storageUpdate(value) {
+    const item = StorageItem.create({ key: storageKey, value: BytesBlob.blobFromString(value) });
+    return {
+        storage: new Map([[tryAsServiceId(0), [UpdateStorage.set({ storage: item })]]]),
+    };
+}
+// > 32 bytes => stored in the values db (not embedded in the leaf).
+const BIG_1 = "a".repeat(100);
+const BIG_2 = "b".repeat(100);
+/** `true` if every value referenced by the state can still be resolved. */
+function canReadFully(state) {
+    if (state === null) {
+        return false;
+    }
+    try {
+        state.backend.intoStateEntries();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Value removals are queued, so poll for the expected outcome. */
+async function eventually(check, what, timeoutMs = 5_000) {
+    const start = Date.now();
+    while (!check()) {
+        if (Date.now() - start > timeoutMs) {
+            assert.fail(`Timed out waiting for: ${what}`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+}
+describe("Fjall hybrid serialized states value refcounting", () => {
+    const spec = tinyChainSpec;
+    let dbPath = "";
+    beforeEach(() => {
+        dbPath = createTempDir();
+    });
+    afterEach(() => {
+        fs.rmSync(dbPath, { recursive: true });
+    });
+    it("removes a replaced value from disk once the replacement finalizes", async () => {
+        const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a handle to the post-1 state, surviving the pruning below
+            const stale1 = states.getState(hh(1));
+            const s1 = states.getState(hh(1));
+            assert.ok(s1 !== null);
+            await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
+            states.commitFinalized([hh(1)]);
+            assert.ok(canReadFully(stale1), "still referenced by the finalized tip");
+            states.commitFinalized([hh(2)]);
+            await eventually(() => !canReadFully(stale1), "replaced value removed from fjall");
+            assert.ok(canReadFully(states.getState(hh(2))), "the new finalized tip stays fully readable");
+        }
+        finally {
+            await states.close();
+        }
+    });
+    it("collects values of a pruned dead fork and keeps the surviving chain intact", async () => {
+        const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a dead fork on top of genesis, inserting a different value
+            const fork = states.getState(hh(0));
+            assert.ok(fork !== null);
+            await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_2));
+            const staleFork = states.getState(hh(0xaa));
+            states.markUnused(hh(0xaa));
+            assert.strictEqual(states.getState(hh(0xaa)), null);
+            await eventually(() => !canReadFully(staleFork), "fork-only value removed from fjall");
+            assert.ok(canReadFully(states.getState(hh(1))), "surviving chain is unaffected");
+        }
+        finally {
+            await states.close();
+        }
+    });
+});

package/packages/jam/database-fjall/index.d.ts

@@ -1,3 +1,5 @@
+export * from "./blocks.js";
 export * from "./hybrid-states.js";
 export * from "./root.js";
+export * from "./states.js";
 //# sourceMappingURL=index.d.ts.map

package/packages/jam/database-fjall/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}