@typeberry/lib 0.10.0-9e3af22 → 0.11.0-411c828

package/packages/jam/database-lmdb/hybrid-states.js

@@ -1,149 +0,0 @@
-import { HashDictionary, SortedSet } from "#@typeberry/collections";
-import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
-import { Logger } from "#@typeberry/logger";
-import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
-import { leafComparator } from "#@typeberry/trie";
-import { OK, Result } from "#@typeberry/utils";
-import { LmdbRoot } from "./root.js";
-const logger = Logger.new(import.meta.filename, "db");
-/**
- * Hybrid serialized-states db.
- *
- * States (leafs) are kept in-memory, but large values are persisted to lmdb.
- * Reads go straight to lmdb, which keeps its own page cache.
- * NOTE: this DB is designed for long fuzzing and to be used with pruning to
- * keep the heap usage bounded.
- *
- * Values that no longer belong to any surviving state are removed from lmdb,
- * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
- * finality signal. Counts are not persisted: this db cannot resume from disk
- * anyway (the leaf sets live in memory), so values left over by a previous
- * run are never collected.
- */
-export class HybridSerializedStates {
-    spec;
-    blake2b;
-    root;
-    inMemStates = HashDictionary.new();
-    lmdbValues;
-    // A single shared values accessor reused by every `LeafDb` we hand out.
-    valuesDb;
-    refsStore = new InMemoryValueRefsStore();
-    refs = new ValueRefs(this.refsStore);
-    // Queue of not-yet-committed value removals, awaited on close.
-    pendingCleanup = Promise.resolve();
-    static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
-        const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
-        return new HybridSerializedStates(spec, blake2b, root);
-    }
-    constructor(spec, blake2b, root) {
-        this.spec = spec;
-        this.blake2b = blake2b;
-        this.root = root;
-        this.lmdbValues = this.root.subDb("values");
-        this.valuesDb = { get: (key) => this.readValue(key) };
-    }
-    async insertInitialState(headerHash, entries) {
-        const { values, leafs } = updateLeafs(SortedSet.fromArray(leafComparator, []), this.blake2b, Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
-        const res = await this.writeValues(values);
-        if (res.isError) {
-            return res;
-        }
-        this.inMemStates.set(headerHash, leafs);
-        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
-        return Result.ok(OK);
-    }
-    async updateAndSetState(header, state, update) {
-        const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
-        // Clone the leaf set before mutating: the previous state keeps using its own.
-        const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
-        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
-        const res = await this.writeValues(values);
-        if (res.isError) {
-            // Leave the caller's state untouched: its new leaves would reference
-            // values that never reached disk.
-            return res;
-        }
-        // Re-create the lookup with the shared values accessor only once the new
-        // values are durably written.
-        state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
-        this.inMemStates.set(header, leafs);
-        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
-        return Result.ok(OK);
-    }
-    async getStateRoot(state) {
-        return state.backend.getStateRoot(this.blake2b);
-    }
-    getState(header) {
-        const leafs = this.inMemStates.get(header);
-        if (leafs === undefined) {
-            return null;
-        }
-        const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
-        return SerializedState.new(this.spec, this.blake2b, leafDb);
-    }
-    commitFinalized(headers) {
-        this.applyRefs(this.refs.commitFinalized(headers));
-    }
-    markUnused(header) {
-        // Release the speculative references first (a no-op for finalized states,
-        // whose deltas were already consumed by `commitFinalized`).
-        this.applyRefs(this.refs.releaseUnfinalized(header));
-        this.inMemStates.delete(header);
-    }
-    /** Apply a refcounting update and remove values that lost their last reference. */
-    applyRefs(update) {
-        this.refsStore.apply(update);
-        if (update.removeValues.length === 0) {
-            return;
-        }
-        // Queued, not awaited: a failed removal only leaks a value.
-        this.pendingCleanup = this.pendingCleanup
-            .then(() => this.lmdbValues.transaction(() => {
-            for (const v of update.removeValues) {
-                this.lmdbValues.remove(v.raw);
-            }
-        }))
-            .catch((e) => {
-            logger.error `Failed to remove unreferenced values: ${e}`;
-        });
-    }
-    diskSizeInBytes() {
-        return this.root.sizeInBytes();
-    }
-    async close() {
-        await this.pendingCleanup;
-        await this.lmdbValues.close();
-        await this.root.close();
-    }
-    /** Write new large values to LMDB in one transaction. */
-    async writeValues(values) {
-        if (values.length === 0) {
-            return Result.ok(OK);
-        }
-        try {
-            // Flush queued removals first: a pending cleanup might be about to delete
-            // a content hash we are re-inserting now. Writing behind it keeps the
-            // re-inserted value on disk (removals are only queued at refcount 0, so
-            // none can be queued for a value still referenced).
-            await this.pendingCleanup;
-            await this.lmdbValues.transaction(() => {
-                for (const [hash, val] of values) {
-                    this.lmdbValues.put(hash.raw, val.raw);
-                }
-            });
-        }
-        catch (e) {
-            return Result.error(StateUpdateError.Commit, () => `Failed to commit values: ${e}`);
-        }
-        return Result.ok(OK);
-    }
-    /** Read a value from LMDB. */
-    readValue(key) {
-        const val = this.lmdbValues.get(key.raw);
-        if (val === undefined) {
-            throw new Error(`Missing value at key: ${key}`);
-        }
-        return val;
-    }
-}

package/packages/jam/database-lmdb/hybrid-states.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=hybrid-states.test.d.ts.map

package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"hybrid-states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.test.ts"],"names":[],"mappings":""}

package/packages/jam/database-lmdb/hybrid-states.test.js

@@ -1,180 +0,0 @@
-// packages/jam/database-lmdb/hybrid-states.test.ts
-import assert from "node:assert";
-import * as fs from "node:fs";
-import { afterEach, before, beforeEach, describe, it } from "node:test";
-import { tryAsServiceId } from "#@typeberry/block";
-import { Bytes, BytesBlob } from "#@typeberry/bytes";
-import { tinyChainSpec } from "#@typeberry/config";
-import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
-import { InMemoryState, StorageItem, UpdateStorage, } from "#@typeberry/state";
-import { testState } from "#@typeberry/state/test.utils.js";
-import { StateEntries } from "#@typeberry/state-merkleization";
-import { asOpaqueType, deepEqual, OK, Result } from "#@typeberry/utils";
-import { HybridSerializedStates } from "./hybrid-states.js";
-let blake2b;
-before(async () => {
-    blake2b = await Blake2b.createHasher();
-});
-function createTempDir(suffix = "hybrid") {
-    return fs.mkdtempSync(`typeberry-${suffix}`);
-}
-describe("Hybrid serialized states", () => {
-    const spec = tinyChainSpec;
-    const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
-    let dbPath = "";
-    beforeEach(() => {
-        dbPath = createTempDir();
-    });
-    afterEach(() => {
-        fs.rmSync(dbPath, { recursive: true });
-    });
-    it("round-trips an initial state through the on-disk values store", async () => {
-        const states = HybridSerializedStates.new({
-            spec,
-            blake2b,
-            dbPath,
-        });
-        try {
-            const empty = InMemoryState.empty(spec);
-            const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
-            const expectedRoot = serialized.getRootHash(blake2b);
-            const res = await states.insertInitialState(headerHash, serialized);
-            deepEqual(res, Result.ok(OK));
-            const state = states.getState(headerHash);
-            assert.ok(state !== null);
-            const stateRoot = await states.getStateRoot(state);
-            assert.strictEqual(`${stateRoot}`, `${expectedRoot}`);
-            deepEqual(InMemoryState.copyFrom(spec, state, new Map()), empty);
-        }
-        finally {
-            await states.close();
-        }
-    });
-    it("reads large values back from disk", async () => {
-        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
-        try {
-            // > 32 bytes => stored in the values db (not embedded in the leaf).
-            const big1 = BytesBlob.blobFromString("x".repeat(100));
-            const big2 = BytesBlob.blobFromString("y".repeat(100));
-            const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
-            const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
-            const entries = StateEntries.fromEntriesUnsafe([
-                [key1, big1],
-                [key2, big2],
-            ]);
-            const res = await states.insertInitialState(headerHash, entries);
-            deepEqual(res, Result.ok(OK));
-            const state = states.getState(headerHash);
-            assert.ok(state !== null);
-            assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
-            assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
-        }
-        finally {
-            await states.close();
-        }
-    });
-    it("drops the leaf set on markUnused while values stay on disk", async () => {
-        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
-        try {
-            const empty = InMemoryState.empty(spec);
-            const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
-            await states.insertInitialState(headerHash, serialized);
-            assert.ok(states.getState(headerHash) !== null);
-            states.markUnused(headerHash);
-            assert.strictEqual(states.getState(headerHash), null);
-        }
-        finally {
-            await states.close();
-        }
-    });
-});
-function hh(n) {
-    return Bytes.fill(HASH_SIZE, n).asOpaque();
-}
-const storageKey = asOpaqueType(BytesBlob.blobFromString("test-key"));
-/** A state update writing a single large (non-embedded) value under `storageKey`. */
-function storageUpdate(value) {
-    const item = StorageItem.create({ key: storageKey, value: BytesBlob.blobFromString(value) });
-    return {
-        storage: new Map([[tryAsServiceId(0), [UpdateStorage.set({ storage: item })]]]),
-    };
-}
-// > 32 bytes => stored in the values db (not embedded in the leaf).
-const BIG_1 = "a".repeat(100);
-const BIG_2 = "b".repeat(100);
-/** `true` if every value referenced by the state can still be resolved. */
-function canReadFully(state) {
-    if (state === null) {
-        return false;
-    }
-    try {
-        state.backend.intoStateEntries();
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-/** Value removals are queued, so poll for the expected outcome. */
-async function eventually(check, what, timeoutMs = 5_000) {
-    const start = Date.now();
-    while (!check()) {
-        if (Date.now() - start > timeoutMs) {
-            assert.fail(`Timed out waiting for: ${what}`);
-        }
-        await new Promise((resolve) => setTimeout(resolve, 10));
-    }
-}
-describe("Hybrid serialized states value refcounting", () => {
-    const spec = tinyChainSpec;
-    let dbPath = "";
-    beforeEach(() => {
-        dbPath = createTempDir();
-    });
-    afterEach(() => {
-        fs.rmSync(dbPath, { recursive: true });
-    });
-    it("removes a replaced value from disk once the replacement finalizes", async () => {
-        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
-        try {
-            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
-            const s0 = states.getState(hh(0));
-            assert.ok(s0 !== null);
-            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
-            // a handle to the post-1 state, surviving the pruning below
-            const stale1 = states.getState(hh(1));
-            const s1 = states.getState(hh(1));
-            assert.ok(s1 !== null);
-            await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
-            states.commitFinalized([hh(1)]);
-            assert.ok(canReadFully(stale1), "still referenced by the finalized tip");
-            states.commitFinalized([hh(2)]);
-            await eventually(() => !canReadFully(stale1), "replaced value removed from lmdb");
-            assert.ok(canReadFully(states.getState(hh(2))), "the new finalized tip stays fully readable");
-        }
-        finally {
-            await states.close();
-        }
-    });
-    it("collects values of a pruned dead fork and keeps the surviving chain intact", async () => {
-        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
-        try {
-            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
-            const s0 = states.getState(hh(0));
-            assert.ok(s0 !== null);
-            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
-            // a dead fork on top of genesis, inserting a different value
-            const fork = states.getState(hh(0));
-            assert.ok(fork !== null);
-            await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_2));
-            const staleFork = states.getState(hh(0xaa));
-            states.markUnused(hh(0xaa));
-            assert.strictEqual(states.getState(hh(0xaa)), null);
-            await eventually(() => !canReadFully(staleFork), "fork-only value removed from lmdb");
-            assert.ok(canReadFully(states.getState(hh(1))), "surviving chain is unaffected");
-        }
-        finally {
-            await states.close();
-        }
-    });
-});

package/packages/jam/database-lmdb/index.d.ts

@@ -1,5 +0,0 @@
-export * from "./blocks.js";
-export * from "./hybrid-states.js";
-export * from "./root.js";
-export * from "./states.js";
-//# sourceMappingURL=index.d.ts.map

package/packages/jam/database-lmdb/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}

package/packages/jam/database-lmdb/index.js

@@ -1,4 +0,0 @@
-export * from "./blocks.js";
-export * from "./hybrid-states.js";
-export * from "./root.js";
-export * from "./states.js";

package/packages/jam/database-lmdb/root.d.ts

@@ -1,26 +0,0 @@
-import * as lmdb from "lmdb";
-export type SubDb = lmdb.Database<Uint8Array, lmdb.Key>;
-export type LmdbRootOptions = {
-    readOnly?: boolean;
-    ephemeral?: boolean;
-    compression?: boolean;
-};
-/** A thin abstraction over lmdb database interface. */
-export declare class LmdbRoot {
-    readonly db: lmdb.RootDatabase<Uint8Array, lmdb.Key>;
-    /** Path of the underlying LMDB data file, used to report on-disk usage. */
-    private readonly dataFilePath;
-    static new(dbPath: string, options: LmdbRootOptions): LmdbRoot;
-    private constructor();
-    /** Open a sub-database under the same path. */
-    subDb(name: string): SubDb;
-    /**
-     * Apparent on-disk size of the LMDB data file, in bytes.
-     *
-     * Returns `null` if the file cannot be `stat`-ed (e.g. not yet created).
-     */
-    sizeInBytes(): number | null;
-    /** Close the database and all sub-databases. */
-    close(): Promise<void>;
-}
-//# sourceMappingURL=root.d.ts.map

package/packages/jam/database-lmdb/root.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/root.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;AAExD,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,uDAAuD;AACvD,qBAAa,QAAQ;IACnB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACrD,2EAA2E;IAC3E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe;IAInD,OAAO;IAwBP,+CAA+C;IAC/C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;IAI1B;;;;OAIG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,gDAAgD;IAC1C,KAAK;CAGZ"}

package/packages/jam/database-lmdb/root.js

@@ -1,53 +0,0 @@
-import * as fs from "node:fs";
-import * as lmdb from "lmdb";
-/** A thin abstraction over lmdb database interface. */
-export class LmdbRoot {
-    db;
-    /** Path of the underlying LMDB data file, used to report on-disk usage. */
-    dataFilePath;
-    static new(dbPath, options) {
-        return new LmdbRoot(dbPath, options);
-    }
-    constructor(dbPath, { readOnly, ephemeral, compression }) {
-        // `lmdb.open` treats an extension-less path as a directory and stores the
-        // environment in `<dbPath>/data.mdb` (next to `lock.mdb`).
-        this.dataFilePath = `${dbPath}/data.mdb`;
-        const isEphemeral = ephemeral ?? false;
-        this.db = lmdb.open(dbPath, {
-            // experimental options
-            noMemInit: true,
-            remapChunks: true,
-            eventTurnBatching: false,
-            // For ephemeral databases (e.g. the fuzz target, which wipes on every reset)
-            // durability is pointless, so we skip fsync and skip compressing the large
-            // per-block leaf blobs. Both are pure overhead there and dominate the cost.
-            // This trades disk space (uncompressed) and crash-durability for speed.
-            compression: compression ?? !isEphemeral,
-            noSync: isEphemeral,
-            keyEncoding: "binary",
-            encoding: "binary",
-            readOnly,
-        });
-    }
-    /** Open a sub-database under the same path. */
-    subDb(name) {
-        return this.db.openDB({ name });
-    }
-    /**
-     * Apparent on-disk size of the LMDB data file, in bytes.
-     *
-     * Returns `null` if the file cannot be `stat`-ed (e.g. not yet created).
-     */
-    sizeInBytes() {
-        try {
-            return fs.statSync(this.dataFilePath).size;
-        }
-        catch {
-            return null;
-        }
-    }
-    /** Close the database and all sub-databases. */
-    async close() {
-        await this.db.close();
-    }
-}

package/packages/jam/database-lmdb/states.d.ts

@@ -1,76 +0,0 @@
-import type { HeaderHash, StateRootHash } from "#@typeberry/block";
-import type { ChainSpec } from "#@typeberry/config";
-import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
-import type { Blake2b } from "#@typeberry/hash";
-import type { ServicesUpdate, State } from "#@typeberry/state";
-import type { StateEntries } from "#@typeberry/state-merkleization";
-import { SerializedState } from "#@typeberry/state-merkleization";
-import { OK, Result } from "#@typeberry/utils";
-import type { LmdbRoot } from "./root.js";
-/**
- * LMDB-backed state storage.
- *
- * Assumptions:
- * 1. For each block we end up with some posterior state.
- * 2. Majority of state entries will stay the same, however there are some that always change.
- * 3. We rarely need `stateRoot`, only needed to verify that we importing a
- *    block on the correct state.
- * 4. We rarely need intermediate trie nodes, only needed for answering warp-sync (CE129)
- *    and producing state proofs.
- *
- * The trie for JAM is designed so that every node takes exactly 64 bytes. Values
- * that don't fit directly into a trie node (larger than 32bytes) are hash-referenced
- * there.
- * Thanks to this, we can estimate that for each and every state, all of the leaf nodes
- * "should not take a lot". Let's assume that we have:
- * 1. 999 services
- * 2. Each service has 50 preimages, so also roughly 50 preimage lookups.
- * 3. On top of that we have 899 storage entries (arbitrary size)
- * 4. That gives us roughly 1000 trie leaves for each service (50 + 50 + 899 + 1(info))
- * 5. We have 15 always-present storage entries (let's round it up to 1k)
- *
- * Hence in total we should have 999 * 1000 + 15 ~ 1M leaves.
- * So for every state, all of the leaves should occupy roughly `64B * 1M = 64MB`.
- * Large values that do not change, are automatically deduplicated via hash-references.
- *
- * It seems very sensible then to:
- * 1. Just store trie leaves for each state.
- * 2. Prune old (finalized) states (otherwise 24h of states = 14400 blocks * 64MB ~ 1TB)
- * 3. Recompute the trie in-memory only if needed.
- *
- * Obviously from just the leaf nodes we can easily obtain the entire state.
- *
- * That's the approach we are implementing here (for now without pruning:)).
- *
- * Since when referencing some state we always think of it in terms of a posterior
- * state produced by some block, we use `HeaderHash` to index the leaf collections.
- *
- * The state root (if needed) can be easily recomputed by merkelizing the leaves.
- *
- * We might need to implement some reference counting for values, or at least
- * know the last state that is referencing them, so that they can be purged from
- * the `values` database when they are not needed any more.
- *
- * TODO [ToDr] To implement when actually needed:
- * - [ ] state pruning on finality / pre-defined window (warp threshold?)
- * - [ ] removing unused values
- */
-export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
-    private readonly spec;
-    private readonly blake2b;
-    private readonly root;
-    private readonly states;
-    private readonly values;
-    static new(spec: ChainSpec, blake2b: Blake2b, root: LmdbRoot): LmdbStates;
-    private constructor();
-    insertInitialState(headerHash: HeaderHash, serializedState: StateEntries): Promise<Result<OK, StateUpdateError>>;
-    private updateAndCommit;
-    updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
-    getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
-    getState(root: HeaderHash): SerializedState<LeafDb> | null;
-    commitFinalized(_headers: HeaderHash[]): void;
-    markUnused(header: HeaderHash): void;
-    diskSizeInBytes(): number | null;
-    close(): Promise<void>;
-}
-//# sourceMappingURL=states.d.ts.map

package/packages/jam/database-lmdb/states.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}

package/packages/jam/database-lmdb/states.js

@@ -1,145 +0,0 @@
-import { BytesBlob } from "#@typeberry/bytes";
-import { SortedSet } from "#@typeberry/collections";
-import { LeafDb, StateUpdateError, updateLeafs } from "#@typeberry/database";
-import { Logger } from "#@typeberry/logger";
-import { SerializedState, StateEntryUpdateAction, serializeStateUpdate } from "#@typeberry/state-merkleization";
-import { leafComparator } from "#@typeberry/trie";
-import { OK, Result, resultToString } from "#@typeberry/utils";
-const logger = Logger.new(import.meta.filename, "db");
-/**
- * LMDB-backed state storage.
- *
- * Assumptions:
- * 1. For each block we end up with some posterior state.
- * 2. Majority of state entries will stay the same, however there are some that always change.
- * 3. We rarely need `stateRoot`, only needed to verify that we importing a
- *    block on the correct state.
- * 4. We rarely need intermediate trie nodes, only needed for answering warp-sync (CE129)
- *    and producing state proofs.
- *
- * The trie for JAM is designed so that every node takes exactly 64 bytes. Values
- * that don't fit directly into a trie node (larger than 32bytes) are hash-referenced
- * there.
- * Thanks to this, we can estimate that for each and every state, all of the leaf nodes
- * "should not take a lot". Let's assume that we have:
- * 1. 999 services
- * 2. Each service has 50 preimages, so also roughly 50 preimage lookups.
- * 3. On top of that we have 899 storage entries (arbitrary size)
- * 4. That gives us roughly 1000 trie leaves for each service (50 + 50 + 899 + 1(info))
- * 5. We have 15 always-present storage entries (let's round it up to 1k)
- *
- * Hence in total we should have 999 * 1000 + 15 ~ 1M leaves.
- * So for every state, all of the leaves should occupy roughly `64B * 1M = 64MB`.
- * Large values that do not change, are automatically deduplicated via hash-references.
- *
- * It seems very sensible then to:
- * 1. Just store trie leaves for each state.
- * 2. Prune old (finalized) states (otherwise 24h of states = 14400 blocks * 64MB ~ 1TB)
- * 3. Recompute the trie in-memory only if needed.
- *
- * Obviously from just the leaf nodes we can easily obtain the entire state.
- *
- * That's the approach we are implementing here (for now without pruning:)).
- *
- * Since when referencing some state we always think of it in terms of a posterior
- * state produced by some block, we use `HeaderHash` to index the leaf collections.
- *
- * The state root (if needed) can be easily recomputed by merkelizing the leaves.
- *
- * We might need to implement some reference counting for values, or at least
- * know the last state that is referencing them, so that they can be purged from
- * the `values` database when they are not needed any more.
- *
- * TODO [ToDr] To implement when actually needed:
- * - [ ] state pruning on finality / pre-defined window (warp threshold?)
- * - [ ] removing unused values
- */
-export class LmdbStates {
-    spec;
-    blake2b;
-    root;
-    states;
-    values;
-    static new(spec, blake2b, root) {
-        return new LmdbStates(spec, blake2b, root);
-    }
-    constructor(spec, blake2b, root) {
-        this.spec = spec;
-        this.blake2b = blake2b;
-        this.root = root;
-        this.states = this.root.subDb("states");
-        this.values = this.root.subDb("values");
-    }
-    async insertInitialState(headerHash, serializedState) {
-        return await this.updateAndCommit(headerHash, SortedSet.fromArray(leafComparator, []), Array.from(serializedState, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
-    }
-    async updateAndCommit(headerHash, leafs, data) {
-        const { values } = updateLeafs(leafs, this.blake2b, data);
-        // TODO [ToDr] could be optimized to already have leaves written to one big chunk
-        // (we could pre-allocate one buffer for all the leafs)
-        const stateLeafs = BytesBlob.blobFromParts(leafs.array.map((x) => x.node.raw));
-        // now we have the leaves and the values, so let's write it down to the DB.
-        const statesWrite = this.states.put(headerHash.raw, stateLeafs.raw);
-        const valuesWrite = this.values.transaction(() => {
-            for (const [hash, val] of values) {
-                this.values.put(hash.raw, val.raw);
-            }
-        });
-        try {
-            await Promise.all([valuesWrite, statesWrite]);
-        }
-        catch (e) {
-            logger.error `${e}`;
-            return Result.error(StateUpdateError.Commit, () => `Failed to commit state update: ${e}`);
-        }
-        return Result.ok(OK);
-    }
-    async updateAndSetState(headerHash, state, update) {
-        const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
-        // and finally we insert new values and store leaves in the DB.
-        const res = await this.updateAndCommit(headerHash, state.backend.leafs, updatedValues);
-        if (res.isOk) {
-            // update the internal backend with new leaves.
-            state.updateBackend(LeafDb.fromLeaves(state.backend.leafs, state.backend.db));
-        }
-        return res;
-    }
-    async getStateRoot(state) {
-        return state.backend.getStateRoot(this.blake2b);
-    }
-    getState(root) {
-        const leafNodes = this.states.get(root.raw);
-        // we don't have that particular state.
-        if (leafNodes === undefined) {
-            return null;
-        }
-        const values = this.values;
-        const leafDbResult = LeafDb.fromLeavesBlob(BytesBlob.blobFrom(leafNodes), {
-            get(key) {
-                const val = values.get(key.raw);
-                if (val === undefined) {
-                    throw new Error(`Missing required value: ${key} in the DB`);
-                }
-                return val;
-            },
-        });
-        if (leafDbResult.isError) {
-            throw new Error(`Inconsistent DB. Invalid leaf nodes for ${root}: ${resultToString(leafDbResult)}`);
-        }
-        return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
-    }
-    commitFinalized(_headers) {
-        // Values are never pruned here. This db survives restarts, so refcounting
-        // would need counts persisted (and crash-consistent) alongside the values.
-        // See the TODO above - not implemented until actually needed.
-    }
-    markUnused(header) {
-        this.states.removeSync(header.raw);
-    }
-    diskSizeInBytes() {
-        return this.root.sizeInBytes();
-    }
-    async close() {
-        await Promise.all([this.states.close(), this.values.close()]);
-    }
-}

package/packages/jam/database-lmdb/states.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=states.test.d.ts.map

package/packages/jam/database-lmdb/states.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.test.ts"],"names":[],"mappings":""}