@typeberry/lib 0.8.4 → 0.9.0-1c3f157

package/packages/jam/database-fjall/states.test.js

@@ -0,0 +1,150 @@
+import assert from "node:assert";
+import * as fs from "node:fs";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { tinyChainSpec } from "#@typeberry/config";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
+import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
+import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
+import { StateEntries } from "#@typeberry/state-merkleization";
+import { deepEqual, OK, Result } from "#@typeberry/utils";
+import { FjallRoot } from "./root.js";
+import { FjallStates } from "./states.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+describe("Fjall states database", () => {
+    const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
+    const spec = tinyChainSpec;
+    let tmpDir = "";
+    beforeEach(() => {
+        tmpDir = fs.mkdtempSync("typeberry-fjall-states-");
+    });
+    afterEach(() => {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+    it("imports and reads an empty state", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const emptyState = InMemoryState.empty(spec);
+            const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
+            const emptyRoot = serialized.getRootHash(blake2b);
+            deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
+            const newState = states.getState(headerHash);
+            assert.ok(newState !== null);
+            assert.strictEqual(`${await states.getStateRoot(newState)}`, `${emptyRoot}`);
+            deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+    it("reads large values back from disk", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const big1 = BytesBlob.blobFromString("x".repeat(100));
+            const big2 = BytesBlob.blobFromString("y".repeat(100));
+            const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
+            const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
+            const entries = StateEntries.fromEntriesUnsafe([
+                [key1, big1],
+                [key2, big2],
+            ]);
+            deepEqual(await states.insertInitialState(headerHash, entries), Result.ok(OK));
+            const state = states.getState(headerHash);
+            assert.ok(state !== null);
+            assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
+            assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+    it("updates state", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const state = InMemoryState.empty(spec);
+            await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
+            const newState = states.getState(headerHash);
+            assert.ok(newState !== null);
+            const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
+            const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
+            const stateUpdate = {
+                timeslot: tryAsTimeSlot(15),
+                privilegedServices: PrivilegedServices.create({
+                    manager: tryAsServiceId(1),
+                    assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
+                    delegator: tryAsServiceId(3),
+                    registrar: tryAsServiceId(4),
+                    autoAccumulateServices: new Map(),
+                }),
+                updated: new Map([
+                    [
+                        tryAsServiceId(1),
+                        UpdateService.create({
+                            serviceInfo: ServiceAccountInfo.create({
+                                codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
+                                balance: tryAsU64(1_000_000),
+                                accumulateMinGas: tryAsServiceGas(10_000),
+                                onTransferMinGas: tryAsServiceGas(5_000),
+                                storageUtilisationBytes: tryAsU64(1_000),
+                                gratisStorage: tryAsU64(0),
+                                storageUtilisationCount: tryAsU32(1),
+                                created: tryAsTimeSlot(0),
+                                lastAccumulation: tryAsTimeSlot(0),
+                                parentService: tryAsServiceId(0),
+                            }),
+                            lookupHistory,
+                        }),
+                    ],
+                ]),
+            };
+            deepEqual(state.applyUpdate(stateUpdate), Result.ok(OK));
+            deepEqual(await states.updateAndSetState(headerHash2, newState, stateUpdate), Result.ok(OK));
+            const updatedState = states.getState(headerHash2);
+            assert.ok(updatedState !== null);
+            deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
+                [
+                    tryAsServiceId(1),
+                    {
+                        storageKeys: [],
+                        preimages: [],
+                        lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
+                    },
+                ],
+            ])), state);
+            assert.strictEqual(`${await states.getStateRoot(updatedState)}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+    it("imports a more complex state", async () => {
+        const root = await FjallRoot.open(tmpDir, { ephemeral: true });
+        const states = await FjallStates.open(spec, blake2b, root);
+        try {
+            const initialState = testState();
+            const initialService = initialState.services.get(tryAsServiceId(0));
+            assert.ok(initialService !== undefined);
+            const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
+            deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
+            const newState = states.getState(headerHash);
+            assert.ok(newState !== null);
+            assert.strictEqual(`${await states.getStateRoot(newState)}`, `${serialized.getRootHash(blake2b)}`);
+            deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
+        }
+        finally {
+            await states.close();
+            await root.close();
+        }
+    });
+});

package/packages/jam/database-lmdb/hybrid-states.d.ts

@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
  * Reads go straight to lmdb, which keeps its own page cache.
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
  * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
  */
 export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
     private readonly spec;
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     private readonly inMemStates;
     private readonly lmdbValues;
     private readonly valuesDb;
+    private readonly refsStore;
+    private readonly refs;
+    private pendingCleanup;
     static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
         spec: ChainSpec;
         blake2b: Blake2b;
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
     updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    private applyRefs;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;
     /** Write new large values to LMDB in one transaction. */

package/packages/jam/database-lmdb/hybrid-states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
+{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-lmdb/hybrid-states.js

@@ -1,9 +1,11 @@
 import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
+import { Logger } from "#@typeberry/logger";
 import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
 import { leafComparator } from "#@typeberry/trie";
 import { OK, Result } from "#@typeberry/utils";
 import { LmdbRoot } from "./root.js";
+const logger = Logger.new(import.meta.filename, "db");
 /**
  * Hybrid serialized-states db.
  *
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
  * Reads go straight to lmdb, which keeps its own page cache.
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
  * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
  */
 export class HybridSerializedStates {
     spec;
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
     lmdbValues;
     // A single shared values accessor reused by every `LeafDb` we hand out.
     valuesDb;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
     static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
         const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
         return new HybridSerializedStates(spec, blake2b, root);
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
             return res;
         }
         this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
         return Result.ok(OK);
     }
     async updateAndSetState(header, state, update) {
         const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
         // Clone the leaf set before mutating: the previous state keeps using its own.
         const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
         const res = await this.writeValues(values);
         if (res.isError) {
             // Leave the caller's state untouched: its new leaves would reference
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
         // values are durably written.
         state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
         this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
         return Result.ok(OK);
     }
     async getStateRoot(state) {
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
         const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
         return SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
     markUnused(header) {
-        // We only remove the state from memory - values are not pruned at all,
-        // but since they are stored on disk we should be safe.
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
         this.inMemStates.delete(header);
     }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => this.lmdbValues.transaction(() => {
+            for (const v of update.removeValues) {
+                this.lmdbValues.remove(v.raw);
+            }
+        }))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
+    }
     diskSizeInBytes() {
         return this.root.sizeInBytes();
     }
     async close() {
+        await this.pendingCleanup;
         await this.lmdbValues.close();
         await this.root.close();
     }
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
             return Result.ok(OK);
         }
         try {
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
             await this.lmdbValues.transaction(() => {
                 for (const [hash, val] of values) {
                     this.lmdbValues.put(hash.raw, val.raw);

package/packages/jam/database-lmdb/states.d.ts

@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
     updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(root: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;

package/packages/jam/database-lmdb/states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
+{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}

package/packages/jam/database-lmdb/states.js

@@ -128,6 +128,11 @@ export class LmdbStates {
         }
         return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
     }
+    commitFinalized(_headers) {
+        // Values are never pruned here. This db survives restarts, so refcounting
+        // would need counts persisted (and crash-consistent) alongside the values.
+        // See the TODO above - not implemented until actually needed.
+    }
     markUnused(header) {
         this.states.removeSync(header.raw);
     }

package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts

@@ -1,6 +1,7 @@
 import type { Epoch } from "#@typeberry/block";
 import type { SignedTicket } from "#@typeberry/block/tickets.js";
 import type { ChainSpec } from "#@typeberry/config";
+import { type TicketValidator } from "#@typeberry/ticket-pool";
 import type { Connections } from "../peers.js";
 import type { StreamManager } from "../stream-manager.js";
 /**
@@ -9,33 +10,40 @@ import type { StreamManager } from "../stream-manager.js";
  * Uses CE-132 (proxy-to-all) for direct broadcast to all peers.
  * Implements a maintain pattern similar to SyncTask: tickets are collected
  * and periodically distributed to peers that haven't received them yet.
+ *
+ * Incoming tickets from peers are first run through a {@link TicketValidator};
+ * only validated tickets are added to the redistribution pool. The default
+ * validator denies everything, so callers must wire a real one via
+ * {@link setTicketValidator} before any networked ticket can be redistributed.
  */
 export declare class TicketDistributionTask {
     private readonly streamManager;
     private readonly connections;
     static start(streamManager: StreamManager, connections: Connections, chainSpec: ChainSpec): TicketDistributionTask;
-    /** Pending tickets waiting to be distributed to peers */
-    private pendingTickets;
-    /** Current epoch being tracked (cleared when epoch changes) */
-    private currentEpoch;
+    private readonly pool;
+    private validator;
     private constructor();
     /**
      * Should be called periodically to distribute pending tickets to connected peers.
      */
     maintainDistribution(): void;
     /**
-     * Add a ticket to the pending queue for distribution.
+     * Add a ticket to the redistribution pool.
      * Clears pending tickets when epoch changes.
      * Deduplicates tickets based on signature.
      */
     addTicket(epochIndex: Epoch, ticket: SignedTicket): void;
-    private onTicketReceivedCallback;
     /**
-     * Register a callback that validates a received ticket.
-     * The ticket is only added to the redistribution pool if the callback returns `true`.
-     * This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
+     * Replace the redistribution pool for the given epoch with the supplied tickets.
+     * Used when the authorship worker dumps the authoritative pool on an epoch boundary.
+     */
+    replacePool(epochIndex: Epoch, tickets: readonly SignedTicket[]): void;
+    /**
+     * Register the validator that decides whether tickets received from peers should be
+     * accepted (and therefore redistributed). The default is {@link DenyTicketsValidator},
+     * so the caller must install a real validator for any peer ticket to make it through.
      */
-    setOnTicketReceived(cb: (epochIndex: Epoch, ticket: SignedTicket) => Promise<boolean>): void;
+    setTicketValidator(validator: TicketValidator): void;
     private onTicketReceived;
 }
 //# sourceMappingURL=ticket-distribution.d.ts.map

package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAuB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAvB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAA0D;IAChF,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAkDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAsCjD,OAAO,CAAC,wBAAwB,CAAgF;IAEhH;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC;IAIrF,OAAO,CAAC,gBAAgB;CAsBzB"}
+{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAA2C,KAAK,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEvG,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAqB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IArB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA2B;IAChD,OAAO,CAAC,SAAS,CAA+C;IAEhE,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAgDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAIjD;;;OAGG;IACH,WAAW,CAAC,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,YAAY,EAAE;IAI/D;;;;OAIG;IACH,kBAAkB,CAAC,SAAS,EAAE,eAAe;IAI7C,OAAO,CAAC,gBAAgB;CAkBzB"}

package/packages/jam/jamnp-s/tasks/ticket-distribution.js

@@ -1,4 +1,5 @@
 import { Logger } from "#@typeberry/logger";
+import { DenyTicketsValidator, PendingTicketPool } from "#@typeberry/ticket-pool";
 import { OK } from "#@typeberry/utils";
 import { ce131 } from "../protocol/index.js";
 const logger = Logger.new(import.meta.filename, "net:tickets");
@@ -12,6 +13,11 @@ const TICKET_AUX = {
  * Uses CE-132 (proxy-to-all) for direct broadcast to all peers.
  * Implements a maintain pattern similar to SyncTask: tickets are collected
  * and periodically distributed to peers that haven't received them yet.
+ *
+ * Incoming tickets from peers are first run through a {@link TicketValidator};
+ * only validated tickets are added to the redistribution pool. The default
+ * validator denies everything, so callers must wire a real one via
+ * {@link setTicketValidator} before any networked ticket can be redistributed.
  */
 export class TicketDistributionTask {
     streamManager;
@@ -26,10 +32,8 @@ export class TicketDistributionTask {
         streamManager.registerOutgoingHandlers(ce131.ClientHandler.new(chainSpec, ce131.STREAM_KIND_PROXY_TO_ALL));
         return task;
     }
-    /** Pending tickets waiting to be distributed to peers */
-    pendingTickets = [];
-    /** Current epoch being tracked (cleared when epoch changes) */
-    currentEpoch = null;
+    pool = new PendingTicketPool();
+    validator = new DenyTicketsValidator();
     constructor(streamManager, connections) {
         this.streamManager = streamManager;
         this.connections = connections;
@@ -38,14 +42,13 @@ export class TicketDistributionTask {
      * Should be called periodically to distribute pending tickets to connected peers.
      */
     maintainDistribution() {
-        if (this.currentEpoch === null) {
-            return; // No tickets to distribute yet
+        const currentEpoch = this.pool.currentEpoch;
+        if (currentEpoch === null) {
+            return;
         }
-        /** `this` is mutable and TS can't narrow this.currentEpoch inside the callback closure */
-        const currentEpoch = this.currentEpoch;
-        // Iterate through all pending tickets
-        for (let ticketIdx = 0; ticketIdx < this.pendingTickets.length; ticketIdx++) {
-            const { epochIndex, ticket } = this.pendingTickets[ticketIdx];
+        const tickets = this.pool.getTickets();
+        for (let ticketIdx = 0; ticketIdx < tickets.length; ticketIdx++) {
+            const { epochIndex, ticket } = tickets[ticketIdx];
             // Try to send to each connected peer
             for (const peerInfo of this.connections.getConnectedPeers()) {
                 this.connections.withAuxData(peerInfo.peerId, TICKET_AUX, (maybeAux) => {
@@ -77,72 +80,45 @@ export class TicketDistributionTask {
         }
     }
     /**
-     * Add a ticket to the pending queue for distribution.
+     * Add a ticket to the redistribution pool.
      * Clears pending tickets when epoch changes.
      * Deduplicates tickets based on signature.
      */
     addTicket(epochIndex, ticket) {
-        // Drop tickets for older epochs (can happen when a delayed validation callback completes
-        // after the epoch has already advanced — accepting it would roll back currentEpoch).
-        if (this.currentEpoch !== null && epochIndex < this.currentEpoch) {
-            return;
-        }
-        // Epoch advanced — clear old tickets
-        if (this.currentEpoch !== null && epochIndex > this.currentEpoch) {
-            logger.log `[addTicket] Epoch changed from ${this.currentEpoch} to ${epochIndex}, clearing ${this.pendingTickets.length} old tickets`;
-            this.pendingTickets = [];
-            // Note: We don't need to clear aux data for all peers here.
-            // The aux data contains the epoch, so maintainDistribution will lazily
-            // reset it when it detects an epoch mismatch. This handles both connected
-            // and disconnected peers correctly.
-        }
-        this.currentEpoch = epochIndex;
-        /**
-         * Deduplicate: check if a ticket with the same signature already exists
-         *
-         * Here we are risking "poisoning" the local pendingTickets - i.e:
-         *  1. The adversary sees a signature and swaps the ticket attempt to something different.
-         *  2. This creates an invalid ticket, but prevents a valid ticket with the same signature from being included and distributed.
-         *
-         * TODO [MaSi]: The poisoning risk should be fixed during implementation of ticket validation.
-         */
-        const isDuplicate = this.pendingTickets.some((pending) => pending.epochIndex === epochIndex && pending.ticket.signature.isEqualTo(ticket.signature));
-        if (!isDuplicate) {
-            this.pendingTickets.push({ epochIndex, ticket });
-            logger.info `[addTicket] Added ticket for epoch ${epochIndex}, total: ${this.pendingTickets.length}`;
-        }
+        this.pool.addTicket(epochIndex, ticket);
     }
-    onTicketReceivedCallback = null;
     /**
-     * Register a callback that validates a received ticket.
-     * The ticket is only added to the redistribution pool if the callback returns `true`.
-     * This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
+     * Replace the redistribution pool for the given epoch with the supplied tickets.
+     * Used when the authorship worker dumps the authoritative pool on an epoch boundary.
      */
-    setOnTicketReceived(cb) {
-        this.onTicketReceivedCallback = cb;
+    replacePool(epochIndex, tickets) {
+        this.pool.replace(epochIndex, tickets);
+    }
+    /**
+     * Register the validator that decides whether tickets received from peers should be
+     * accepted (and therefore redistributed). The default is {@link DenyTicketsValidator},
+     * so the caller must install a real validator for any peer ticket to make it through.
+     */
+    setTicketValidator(validator) {
+        this.validator = validator;
     }
     onTicketReceived(epochIndex, ticket) {
         logger.trace `Received ticket for epoch ${epochIndex}, attempt ${ticket.attempt}`;
-        if (this.onTicketReceivedCallback !== null) {
-            // Validate first; only redistribute if valid to avoid spreading tampered tickets.
-            // Wrap with Promise.resolve().then() to catch both sync throws and async rejections.
-            const cb = this.onTicketReceivedCallback;
-            Promise.resolve()
-                .then(() => cb(epochIndex, ticket))
-                .then((isValid) => {
-                if (isValid) {
-                    this.addTicket(epochIndex, ticket);
-                }
-                else {
-                    logger.warn `Dropping invalid ticket for epoch ${epochIndex} (validation failed)`;
-                }
-            })
-                .catch((error) => {
-                logger.error `Error validating ticket for epoch ${epochIndex}, attempt ${ticket.attempt}: ${error}`;
-            });
-        }
-        else {
-            this.addTicket(epochIndex, ticket);
-        }
+        const validator = this.validator;
+        // Wrap with Promise.resolve().then() so a synchronous throw inside the validator
+        // funnels into the same .catch() as an async rejection.
+        Promise.resolve()
+            .then(() => validator.validate(epochIndex, [ticket]))
+            .then((result) => {
+            if (result.isOk) {
+                this.addTicket(epochIndex, ticket);
+            }
+            else {
+                logger.trace `Dropping ticket for epoch ${epochIndex}: ${result.error}`;
+            }
+        })
+            .catch((error) => {
+            logger.error `Error validating ticket for epoch ${epochIndex}, attempt ${ticket.attempt}: ${error}`;
+        });
     }
 }