@typeberry/lib 0.5.1 → 0.5.2-03e1aed

package/packages/core/telemetry/index.js

@@ -0,0 +1,99 @@
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+import { Logger } from "#@typeberry/logger";
+import { env } from "#@typeberry/utils";
+import packageJson from "./package.json" with { type: "json" };
+const logger = Logger.new(import.meta.filename, "tele");
+export class Telemetry {
+    sdk;
+    static initialize(config) {
+        const sdk = initializeTelemetry({
+            isMain: config.isMain ?? false,
+            serviceName: `typeberry-${config.nodeName}`,
+            serviceVersion: packageJson.version,
+            enabled: env.OTEL_ENABLED !== "false",
+            otlpEndpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://localhost:9090/api/v1/otlp",
+            resourceAttributes: {
+                "worker.type": config.worker,
+            },
+        });
+        if (sdk === null) {
+            return null;
+        }
+        return new Telemetry(sdk);
+    }
+    constructor(sdk) {
+        this.sdk = sdk;
+    }
+    async close() {
+        try {
+            await this.sdk.shutdown();
+            logger.trace `📳 OpenTelemetry shut down successfully`;
+        }
+        catch (error) {
+            logger.error `🔴 Error shutting down OpenTelemetry: ${error}`;
+        }
+    }
+}
+function initializeTelemetry(config) {
+    // Check if telemetry is disabled
+    if (config.enabled === false) {
+        logger.info `📳 OpenTelemetry disabled`;
+        return null;
+    }
+    const { serviceName, serviceVersion, otlpEndpoint, resourceAttributes = {} } = config;
+    // Create resource with service information
+    const customResource = resourceFromAttributes({
+        [ATTR_SERVICE_NAME]: serviceName,
+        [ATTR_SERVICE_VERSION]: serviceVersion,
+        ...resourceAttributes,
+    });
+    const resource = defaultResource().merge(customResource);
+    // Setup metric exporters
+    const metricReaders = [];
+    // NOTE [ToDr] We would probably want to expose prometheus metrics endpoint
+    // as well, however because the we use multiple worker threads we need a way
+    // to collect metrics across all of these threads. One way is for each thread
+    // to expose a separate endpoint on different port. Another to pass the
+    // metrics somehow to single thread and aggregate them there.
+    // We will have similar problem with TART (JIP-3) later, so we can work on
+    // shared solution.
+    // For now we are only able to push metrics to the server, and each thread
+    // can do that separately, since it will be aggregated anyway.
+    // OTLP exporter (push-based)
+    const otlpExporter = new OTLPMetricExporter({
+        url: `${otlpEndpoint}/v1/metrics`,
+    });
+    const otlpReader = new PeriodicExportingMetricReader({
+        exporter: otlpExporter,
+        exportIntervalMillis: 3_000,
+    });
+    metricReaders.push(otlpReader);
+    // Initialize the SDK
+    const sdk = new NodeSDK({
+        resource,
+        metricReaders,
+        instrumentations: [
+            getNodeAutoInstrumentations({
+                // Disable specific instrumentations if needed
+                "@opentelemetry/instrumentation-fs": {
+                    enabled: false, // File system instrumentation can be noisy
+                },
+            }),
+        ],
+    });
+    try {
+        sdk.start();
+        if (config.isMain) {
+            logger.info `📳 OTLP metrics will be exported to ${otlpEndpoint}`;
+        }
+    }
+    catch (error) {
+        logger.error `🔴 Error initializing OpenTelemetry: ${error}`;
+    }
+    return sdk;
+}

package/packages/core/telemetry/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "@typeberry/telemetry",
+    "version": "0.5.2",
+    "description": "OpenTelemetry initialization utilities for Typeberry",
+    "license": "MPL-2.0",
+    "author": "Fluffy Labs",
+    "main": "index.ts",
+    "type": "module",
+    "dependencies": {
+        "@opentelemetry/auto-instrumentations-node": "^0.67.0",
+        "@opentelemetry/exporter-metrics-otlp-http": "^0.208.0",
+        "@opentelemetry/resources": "^2.2.0",
+        "@opentelemetry/sdk-metrics": "^2.2.0",
+        "@opentelemetry/sdk-node": "^0.208.0",
+        "@opentelemetry/semantic-conventions": "^1.38.0",
+        "@typeberry/logger": "*",
+        "@typeberry/utils": "*"
+    }
+}

package/packages/core/trie/hasher.d.ts

@@ -0,0 +1,6 @@
+import type { Blake2b } from "#@typeberry/hash";
+import { type KeccakHasher } from "#@typeberry/hash/keccak.js";
+import type { TrieHasher } from "./nodesDb.js";
+export declare function getBlake2bTrieHasher(hasher: Blake2b): TrieHasher;
+export declare function getKeccakTrieHasher(hasher: KeccakHasher): TrieHasher;
+//# sourceMappingURL=hasher.d.ts.map

package/packages/core/trie/hasher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hasher.d.ts","sourceRoot":"","sources":["../../../../../packages/core/trie/hasher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAa,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEzE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/C,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,UAAU,CAMhE;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,GAAG,UAAU,CAMpE"}

package/packages/core/trie/hasher.js

@@ -0,0 +1,16 @@
+import { BytesBlob } from "#@typeberry/bytes";
+import { hashBlobs } from "#@typeberry/hash/keccak.js";
+export function getBlake2bTrieHasher(hasher) {
+    return {
+        hashConcat(n, rest = []) {
+            return hasher.hashBlobs([n, ...rest]);
+        },
+    };
+}
+export function getKeccakTrieHasher(hasher) {
+    return {
+        hashConcat(n, rest = []) {
+            return hashBlobs(hasher, [n, ...rest].map(BytesBlob.blobFrom)).asOpaque();
+        },
+    };
+}

package/packages/core/trie/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./nodes.js";
+export * from "./nodesDb.js";
+export * from "./trie.js";
+//# sourceMappingURL=index.d.ts.map

package/packages/core/trie/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/core/trie/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC"}

package/packages/core/trie/index.js

@@ -0,0 +1,3 @@
+export * from "./nodes.js";
+export * from "./nodesDb.js";
+export * from "./trie.js";

package/packages/core/trie/masks.d.ts

@@ -0,0 +1,5 @@
+export declare const FIRST_BIT_SET = 128;
+export declare const FIRST_BIT_SET_NEG = 127;
+export declare const FIRST_TWO_BITS_SET = 192;
+export declare const FIRST_TWO_BITS_SET_NEG = 63;
+//# sourceMappingURL=masks.d.ts.map

package/packages/core/trie/masks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"masks.d.ts","sourceRoot":"","sources":["../../../../../packages/core/trie/masks.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa,MAAgB,CAAC;AAC3C,eAAO,MAAM,iBAAiB,MAAgB,CAAC;AAE/C,eAAO,MAAM,kBAAkB,MAAgB,CAAC;AAChD,eAAO,MAAM,sBAAsB,KAAgB,CAAC"}

package/packages/core/trie/masks.js

@@ -0,0 +1,4 @@
+export const FIRST_BIT_SET = 0b10_00_00_00;
+export const FIRST_BIT_SET_NEG = 0b01_11_11_11;
+export const FIRST_TWO_BITS_SET = 0b11_00_00_00;
+export const FIRST_TWO_BITS_SET_NEG = 0b00_11_11_11;

package/packages/core/trie/nodes.d.ts

@@ -0,0 +1,137 @@
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { type OpaqueHash } from "#@typeberry/hash";
+import { type Opaque } from "#@typeberry/utils";
+export type StateKey = Opaque<OpaqueHash, "trieStateKey">;
+export type TruncatedStateKey = Opaque<Bytes<TRUNCATED_KEY_BYTES>, "trieStateKey">;
+export type InputKey = StateKey | TruncatedStateKey;
+/**
+ * Hash of the entire node of the trie or concatenation of two nodes.
+ *
+ * In case this is the root node of the entire trie, it's going to be the state commitment.
+ *
+ * https://graypaper.fluffylabs.dev/#/579bd12/0c1f010c2301
+ */
+export type TrieNodeHash = Opaque<OpaqueHash, "trie">;
+/** Hash of the value contained in the trie node. */
+export type ValueHash = Opaque<OpaqueHash, "trieValue">;
+/** Value nodes have the key truncated to 31 bytes. */
+export declare const TRUNCATED_KEY_BYTES = 31;
+export type TRUNCATED_KEY_BYTES = 31;
+export declare const TRUNCATED_KEY_BITS: number;
+/** Number of bytes used to represent a trie node. */
+export declare const TRIE_NODE_BYTES = 64;
+export declare function parseInputKey(v: string): InputKey;
+/**
+ * The kind of the trie node.
+ */
+export declare enum NodeType {
+    /** Branch node (left & right subtree hashes) */
+    Branch = 0,
+    /** Leaf node (value hash) */
+    Leaf = 1,
+    /** Embedded leaf node (value len + value) */
+    EmbedLeaf = 2
+}
+/**
+ * A representation of an unidentified raw trie node.
+ *
+ * The node can be either (determined by the first bit):
+ *	- a branch node
+ *	- a leaf node
+ *
+ * In case of a branch node the contained data is:
+ *	- left sub-node hash (32 bytes - 1 bit)
+ *	- right sub-node hash (32 bytes)
+ *
+ * There are two kinds of leaf nodes (determined by the second bit)
+ *	- Embedded value leaf nodes
+ *	- Value hash leaf nodes
+ *
+ * Embedded value leaf nodes contain:
+ *  - a length of the embedded value (last 6 bits of the first byte)
+ *  - the value itself (padded with zeroes)
+ *
+ * Regular value leaf nodes contain:
+ *  - a hash of the value
+ */
+export declare class TrieNode {
+    /** Exactly 512 bits / 64 bytes */
+    readonly raw: Uint8Array;
+    constructor(
+    /** Exactly 512 bits / 64 bytes */
+    raw?: Uint8Array);
+    /** Returns the type of the node */
+    getNodeType(): NodeType;
+    /** View this node as a branch node */
+    asBranchNode(): BranchNode;
+    /** View this node as a leaf node */
+    asLeafNode(): LeafNode;
+    toString(): string;
+}
+/**
+ * A branch node view of the underlying raw trie node.
+ *
+ * +---------------------------------------------------------------+
+ * |                        512-bit trie node                      |
+ * +---+----------------------------+------------------------------+
+ * | B | Left Sub-node Hash         | Right Sub-node Hash          |
+ * |   | (255 bits)                 | (256 bits)                   |
+ * |---|----------------------------|------------------------------|
+ * | 0 | 101010101010101010101...   | 11001100110011001100...      |
+ * +---------------------------------------------------------------+
+ */
+export declare class BranchNode {
+    readonly node: TrieNode;
+    constructor(node: TrieNode);
+    static fromSubNodes(left: TrieNodeHash, right: TrieNodeHash): BranchNode;
+    /** Get the hash of the left sub-trie. */
+    getLeft(): TrieNodeHash;
+    /** Get the hash of the right sub-trie. */
+    getRight(): TrieNodeHash;
+}
+/**
+ * A leaf node view of the underlying raw trie node.
+ *
+ * +---------------------------------------------------------------+
+ * |                    Embedded value leaf                        |
+ * +----+----------+-------------------+---------------------------+
+ * | BL | V_len    | Key               | 0-padded value (V_len)    |
+ * | 2b | (6 bits) | (31 bytes)        | (32 bytes)                |
+ * |----|----------|-------------------|---------------------------|
+ * | 10 |  000111  | deadbeef...       | 0123456789abcdef...       |
+ * +---------------------------------------------------------------+
+ * |                    Value hash leaf                            |
+ * +----+----------+-------------------+---------------------------+
+ * | BL |   zero   | Key               | Value hash                |
+ * |----|----------|-------------------|---------------------------|
+ * | 11 |  000000  | deadbeef...       | deadbeef...               |
+ * +---------------------------------------------------------------+
+ */
+export declare class LeafNode {
+    readonly node: TrieNode;
+    constructor(node: TrieNode);
+    static fromValue(key: InputKey, value: BytesBlob, valueHash: () => ValueHash): LeafNode;
+    /** Get the key (truncated to 31 bytes). */
+    getKey(): TruncatedStateKey;
+    hasEmbeddedValue(): boolean;
+    /**
+     * Get the byte length of embedded value.
+     *
+     * Note in case this node only contains hash this is going to be 0.
+     */
+    getValueLength(): number;
+    /**
+     * Returns the embedded value.
+     *
+     * Note that this is going to be empty for a regular leaf node (i.e. containing a hash).
+     */
+    getValue(): BytesBlob;
+    /**
+     * Returns contained value hash.
+     *
+     * Note that for embedded value this is going to be full 0-padded 32 bytes.
+     */
+    getValueHash(): ValueHash;
+    toString(): string;
+}
+//# sourceMappingURL=nodes.d.ts.map

package/packages/core/trie/nodes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../../../../packages/core/trie/nodes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAa,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAS,KAAK,MAAM,EAAuB,MAAM,kBAAkB,CAAC;AAG3E,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;AAC1D,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,cAAc,CAAC,CAAC;AAEnF,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,iBAAiB,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;AAEtD,oDAAoD;AACpD,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;AAExD,sDAAsD;AACtD,eAAO,MAAM,mBAAmB,KAAK,CAAC;AACtC,MAAM,MAAM,mBAAmB,GAAG,EAAE,CAAC;AACrC,eAAO,MAAM,kBAAkB,QAA0B,CAAC;AAE1D,qDAAqD;AACrD,eAAO,MAAM,eAAe,KAAK,CAAC;AAElC,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,GAAG,QAAQ,CAKjD;AAED;;GAEG;AACH,oBAAY,QAAQ;IAClB,gDAAgD;IAChD,MAAM,IAAI;IACV,6BAA6B;IAC7B,IAAI,IAAI;IACR,6CAA6C;IAC7C,SAAS,IAAI;CACd;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,QAAQ;IAEjB,kCAAkC;aAClB,GAAG,EAAE,UAAU;;IAD/B,kCAAkC;IAClB,GAAG,GAAE,UAAiD;IAGxE,mCAAmC;IACnC,WAAW,IAAI,QAAQ;IAYvB,sCAAsC;IACtC,YAAY,IAAI,UAAU;IAK1B,oCAAoC;IACpC,UAAU,IAAI,QAAQ;IAKtB,QAAQ;CAGT;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,UAAU;IAET,QAAQ,CAAC,IAAI,EAAE,QAAQ;gBAAd,IAAI,EAAE,QAAQ;IAEnC,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY;IAW3D,yCAAyC;IACzC,OAAO,IAAI,YAAY;IAIvB,0CAA0C;IAC1C,QAAQ,IAAI,YAAY;CAGzB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,QAAQ;IAEnB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;gBAEZ,IAAI,EAAE,QAAQ;IAI1B,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,GAAG,QAAQ;IAoBvF,2CAA2C;IAC3C,MAAM,IAAI,iBAAiB;IAI3B,gBAAgB,IAAI,OAAO;IAI3B;;;;OAIG;IACH,cAAc,IAAI,MAAM;IAMxB;;;;OAIG;IACH,QAAQ,IAAI,SAAS;IAKrB;;;;OAIG;IACH,YAAY,IAAI,SAAS;IAIzB,QAAQ;CAGT"}

package/packages/core/trie/nodes.js

@@ -0,0 +1,196 @@
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { HASH_SIZE } from "#@typeberry/hash";
+import { check, safeAllocUint8Array } from "#@typeberry/utils";
+import { FIRST_BIT_SET, FIRST_BIT_SET_NEG, FIRST_TWO_BITS_SET, FIRST_TWO_BITS_SET_NEG } from "./masks.js";
+/** Value nodes have the key truncated to 31 bytes. */
+export const TRUNCATED_KEY_BYTES = 31;
+export const TRUNCATED_KEY_BITS = TRUNCATED_KEY_BYTES * 8;
+/** Number of bytes used to represent a trie node. */
+export const TRIE_NODE_BYTES = 64;
+export function parseInputKey(v) {
+    if (v.length === HASH_SIZE * 2) {
+        return Bytes.parseBytesNoPrefix(v, HASH_SIZE).asOpaque();
+    }
+    return Bytes.parseBytesNoPrefix(v, TRUNCATED_KEY_BYTES).asOpaque();
+}
+/**
+ * The kind of the trie node.
+ */
+export var NodeType;
+(function (NodeType) {
+    /** Branch node (left & right subtree hashes) */
+    NodeType[NodeType["Branch"] = 0] = "Branch";
+    /** Leaf node (value hash) */
+    NodeType[NodeType["Leaf"] = 1] = "Leaf";
+    /** Embedded leaf node (value len + value) */
+    NodeType[NodeType["EmbedLeaf"] = 2] = "EmbedLeaf";
+})(NodeType || (NodeType = {}));
+/**
+ * A representation of an unidentified raw trie node.
+ *
+ * The node can be either (determined by the first bit):
+ *	- a branch node
+ *	- a leaf node
+ *
+ * In case of a branch node the contained data is:
+ *	- left sub-node hash (32 bytes - 1 bit)
+ *	- right sub-node hash (32 bytes)
+ *
+ * There are two kinds of leaf nodes (determined by the second bit)
+ *	- Embedded value leaf nodes
+ *	- Value hash leaf nodes
+ *
+ * Embedded value leaf nodes contain:
+ *  - a length of the embedded value (last 6 bits of the first byte)
+ *  - the value itself (padded with zeroes)
+ *
+ * Regular value leaf nodes contain:
+ *  - a hash of the value
+ */
+export class TrieNode {
+    raw;
+    constructor(
+    /** Exactly 512 bits / 64 bytes */
+    raw = safeAllocUint8Array(TRIE_NODE_BYTES)) {
+        this.raw = raw;
+    }
+    /** Returns the type of the node */
+    getNodeType() {
+        if ((this.raw[0] & FIRST_BIT_SET) === 0) {
+            return NodeType.Branch;
+        }
+        if ((this.raw[0] & FIRST_TWO_BITS_SET) === FIRST_TWO_BITS_SET) {
+            return NodeType.Leaf;
+        }
+        return NodeType.EmbedLeaf;
+    }
+    /** View this node as a branch node */
+    asBranchNode() {
+        check `${this.getNodeType() === NodeType.Branch} not a branch!`;
+        return new BranchNode(this);
+    }
+    /** View this node as a leaf node */
+    asLeafNode() {
+        check `${this.getNodeType() !== NodeType.Branch} not a leaf!`;
+        return new LeafNode(this);
+    }
+    toString() {
+        return BytesBlob.blobFrom(this.raw).toString();
+    }
+}
+/**
+ * A branch node view of the underlying raw trie node.
+ *
+ * +---------------------------------------------------------------+
+ * |                        512-bit trie node                      |
+ * +---+----------------------------+------------------------------+
+ * | B | Left Sub-node Hash         | Right Sub-node Hash          |
+ * |   | (255 bits)                 | (256 bits)                   |
+ * |---|----------------------------|------------------------------|
+ * | 0 | 101010101010101010101...   | 11001100110011001100...      |
+ * +---------------------------------------------------------------+
+ */
+export class BranchNode {
+    node;
+    // Underlying raw node.
+    constructor(node) {
+        this.node = node;
+    }
+    static fromSubNodes(left, right) {
+        const node = new TrieNode();
+        node.raw.set(left.raw, 0);
+        node.raw.set(right.raw, HASH_SIZE);
+        // set the first bit to 0 (branch node)
+        node.raw[0] &= FIRST_BIT_SET_NEG;
+        return new BranchNode(node);
+    }
+    /** Get the hash of the left sub-trie. */
+    getLeft() {
+        return Bytes.fromBlob(this.node.raw.subarray(0, HASH_SIZE), HASH_SIZE).asOpaque();
+    }
+    /** Get the hash of the right sub-trie. */
+    getRight() {
+        return Bytes.fromBlob(this.node.raw.subarray(HASH_SIZE), HASH_SIZE).asOpaque();
+    }
+}
+/**
+ * A leaf node view of the underlying raw trie node.
+ *
+ * +---------------------------------------------------------------+
+ * |                    Embedded value leaf                        |
+ * +----+----------+-------------------+---------------------------+
+ * | BL | V_len    | Key               | 0-padded value (V_len)    |
+ * | 2b | (6 bits) | (31 bytes)        | (32 bytes)                |
+ * |----|----------|-------------------|---------------------------|
+ * | 10 |  000111  | deadbeef...       | 0123456789abcdef...       |
+ * +---------------------------------------------------------------+
+ * |                    Value hash leaf                            |
+ * +----+----------+-------------------+---------------------------+
+ * | BL |   zero   | Key               | Value hash                |
+ * |----|----------|-------------------|---------------------------|
+ * | 11 |  000000  | deadbeef...       | deadbeef...               |
+ * +---------------------------------------------------------------+
+ */
+export class LeafNode {
+    // Underlying raw node.
+    node;
+    constructor(node) {
+        this.node = node;
+    }
+    static fromValue(key, value, valueHash) {
+        const node = new TrieNode();
+        // The value will fit in the leaf itself.
+        if (value.length <= HASH_SIZE) {
+            node.raw[0] = FIRST_BIT_SET | value.length;
+            // truncate & copy the key
+            node.raw.set(key.raw.subarray(0, TRUNCATED_KEY_BYTES), 1);
+            // copy the value
+            node.raw.set(value.raw, TRUNCATED_KEY_BYTES + 1);
+        }
+        else {
+            node.raw[0] = FIRST_TWO_BITS_SET;
+            // truncate & copy the key
+            node.raw.set(key.raw.subarray(0, TRUNCATED_KEY_BYTES), 1);
+            // copy the value hash
+            node.raw.set(valueHash().raw, TRUNCATED_KEY_BYTES + 1);
+        }
+        return new LeafNode(node);
+    }
+    /** Get the key (truncated to 31 bytes). */
+    getKey() {
+        return Bytes.fromBlob(this.node.raw.subarray(1, TRUNCATED_KEY_BYTES + 1), TRUNCATED_KEY_BYTES).asOpaque();
+    }
+    hasEmbeddedValue() {
+        return this.node.getNodeType() === NodeType.EmbedLeaf;
+    }
+    /**
+     * Get the byte length of embedded value.
+     *
+     * Note in case this node only contains hash this is going to be 0.
+     */
+    getValueLength() {
+        const firstByte = this.node.raw[0];
+        // we only store values up to `HASH_SIZE`, so they fit on the last 6 bits.
+        return firstByte & FIRST_TWO_BITS_SET_NEG;
+    }
+    /**
+     * Returns the embedded value.
+     *
+     * Note that this is going to be empty for a regular leaf node (i.e. containing a hash).
+     */
+    getValue() {
+        const len = this.getValueLength();
+        return BytesBlob.blobFrom(this.node.raw.subarray(HASH_SIZE, HASH_SIZE + len));
+    }
+    /**
+     * Returns contained value hash.
+     *
+     * Note that for embedded value this is going to be full 0-padded 32 bytes.
+     */
+    getValueHash() {
+        return Bytes.fromBlob(this.node.raw.subarray(HASH_SIZE), HASH_SIZE).asOpaque();
+    }
+    toString() {
+        return `LeafNode {\n key: ${this.getKey()},\n valueHash: ${this.getValueHash()}\n}`;
+    }
+}

package/packages/core/trie/nodesDb.d.ts

@@ -0,0 +1,31 @@
+import { HashDictionary } from "#@typeberry/collections";
+import { type LeafNode, type TrieNode, type TrieNodeHash } from "./nodes.js";
+/** Hasher used for the trie nodes. */
+export type TrieHasher = {
+    hashConcat(n: Uint8Array, r?: Uint8Array[]): TrieNodeHash;
+};
+/** An abstraction over read-only nodes storage. */
+export declare class NodesDb {
+    readonly hasher: TrieHasher;
+    protected readonly nodes: HashDictionary<TrieNodeHash, TrieNode>;
+    constructor(hasher: TrieHasher);
+    get(hash: TrieNodeHash): TrieNode | null;
+    hashNode(n: TrieNode): TrieNodeHash;
+    leaves(): Generator<LeafNode>;
+    /**
+     * Returns a string identifier of that hash to be used as a key in DB.
+     *
+     * Before calling `toString` the first bit is set to 0, to maintain compatibility
+     * with branch nodes, which have the left subtree stripped out of the first bit
+     * (since it's a branch node identifier).
+     */
+    protected static withHashCompat<T>(hash: TrieNodeHash, exe: (hash: TrieNodeHash) => T): T;
+}
+/**
+ * A version of `NodesDb` augmented with mutating methods.
+ */
+export declare class WriteableNodesDb extends NodesDb {
+    remove(hash: TrieNodeHash): void;
+    insert(node: TrieNode, hash?: TrieNodeHash): TrieNodeHash;
+}
+//# sourceMappingURL=nodesDb.d.ts.map

package/packages/core/trie/nodesDb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nodesDb.d.ts","sourceRoot":"","sources":["../../../../../packages/core/trie/nodesDb.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD,OAAO,EAAE,KAAK,QAAQ,EAAY,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvF,sCAAsC;AACtC,MAAM,MAAM,UAAU,GAAG;IACvB,UAAU,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC;CAC3D,CAAC;AAEF,mDAAmD;AACnD,qBAAa,OAAO;aAGU,MAAM,EAAE,UAAU;IAF9C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAwB;gBAE5D,MAAM,EAAE,UAAU;IAE9C,GAAG,CAAC,IAAI,EAAE,YAAY,GAAG,QAAQ,GAAG,IAAI;IAMxC,QAAQ,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY;IAIlC,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC;IAU9B;;;;;;OAMG;IACH,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,YAAY,KAAK,CAAC,GAAG,CAAC;CAS1F;AAED;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,OAAO;IAC3C,MAAM,CAAC,IAAI,EAAE,YAAY;IAMzB,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY;CAO1D"}

package/packages/core/trie/nodesDb.js

@@ -0,0 +1,61 @@
+import { HashDictionary } from "#@typeberry/collections";
+import { FIRST_BIT_SET_NEG } from "./masks.js";
+import { NodeType } from "./nodes.js";
+/** An abstraction over read-only nodes storage. */
+export class NodesDb {
+    hasher;
+    nodes = HashDictionary.new();
+    constructor(hasher) {
+        this.hasher = hasher;
+    }
+    get(hash) {
+        return NodesDb.withHashCompat(hash, (key) => {
+            return this.nodes.get(key) ?? null;
+        });
+    }
+    hashNode(n) {
+        return this.hasher.hashConcat(n.raw);
+    }
+    *leaves() {
+        // TODO [ToDr] Perhaps we could avoid iterating over all nodes?
+        for (const val of this.nodes.values()) {
+            const nodeType = val.getNodeType();
+            if (nodeType !== NodeType.Branch) {
+                yield val.asLeafNode();
+            }
+        }
+    }
+    /**
+     * Returns a string identifier of that hash to be used as a key in DB.
+     *
+     * Before calling `toString` the first bit is set to 0, to maintain compatibility
+     * with branch nodes, which have the left subtree stripped out of the first bit
+     * (since it's a branch node identifier).
+     */
+    static withHashCompat(hash, exe) {
+        const prevValue = hash.raw[0];
+        hash.raw[0] &= FIRST_BIT_SET_NEG;
+        const returnValue = exe(hash);
+        // restore the original byte, so that we have correct value in case it
+        // ends up in the right part of the subtree.
+        hash.raw[0] = prevValue;
+        return returnValue;
+    }
+}
+/**
+ * A version of `NodesDb` augmented with mutating methods.
+ */
+export class WriteableNodesDb extends NodesDb {
+    remove(hash) {
+        return NodesDb.withHashCompat(hash, (key) => {
+            this.nodes.delete(key);
+        });
+    }
+    insert(node, hash) {
+        const h = hash ?? this.hashNode(node);
+        NodesDb.withHashCompat(h, (key) => {
+            this.nodes.set(key, node);
+        });
+        return h;
+    }
+}

package/packages/core/trie/trie.d.ts

@@ -0,0 +1,35 @@
+import { type BytesBlob } from "#@typeberry/bytes";
+import type { SortedSet } from "#@typeberry/collections";
+import { type InputKey, LeafNode, type StateKey, type TrieNode, type TrieNodeHash, type TruncatedStateKey, type ValueHash } from "./nodes.js";
+import { type TrieHasher, WriteableNodesDb } from "./nodesDb.js";
+/** Compare two trie `LeafNode`s only by their key. */
+export declare const leafComparator: (x: LeafNode, y: LeafNode) => import("@typeberry/ordering").Ordering;
+export declare class InMemoryTrie {
+    readonly nodes: WriteableNodesDb;
+    private root;
+    /** Create an empty in-memory trie. */
+    static empty(hasher: TrieHasher): InMemoryTrie;
+    /** Given a collection of leaves, compute the state root. */
+    static computeStateRoot(hasher: TrieHasher, leaves: SortedSet<LeafNode>): TrieNodeHash;
+    /**
+     * Construct a `LeafNode` from given `key` and `value`.
+     *
+     * NOTE: for large value it WILL NOT be embedded in the leaf node,
+     * and should rather be stored separately.
+     */
+    static constructLeaf(hasher: TrieHasher, key: InputKey, value: BytesBlob, maybeValueHash?: ValueHash): LeafNode;
+    /**
+     * Reconstruct the entire trie from it's leaves.
+     *
+     * Note that if only the state root is needed, this is rather inefficient.
+     */
+    static fromLeaves(hasher: TrieHasher, leaves: readonly LeafNode[]): InMemoryTrie;
+    private constructor();
+    set(key: InputKey, value: BytesBlob, maybeValueHash?: ValueHash): LeafNode;
+    remove(_: StateKey): void;
+    getRootNode(): TrieNode | null;
+    getRootHash(): TrieNodeHash;
+    toString(): string;
+}
+export declare function findSharedPrefix(a: TruncatedStateKey, b: TruncatedStateKey): number;
+//# sourceMappingURL=trie.d.ts.map

package/packages/core/trie/trie.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trie.d.ts","sourceRoot":"","sources":["../../../../../packages/core/trie/trie.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAGxD,OAAO,EAEL,KAAK,QAAQ,EACb,QAAQ,EAER,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAgB,KAAK,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAE/E,sDAAsD;AACtD,eAAO,MAAM,cAAc,GAAI,GAAG,QAAQ,EAAE,GAAG,QAAQ,2CAAmC,CAAC;AAG3F,qBAAa,YAAY;aA2GL,KAAK,EAAE,gBAAgB;IACvC,OAAO,CAAC,IAAI;IA3Gd,sCAAsC;IACtC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY;IAI9C,4DAA4D;IAC5D,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,YAAY;IAwEtF;;;;;OAKG;IACH,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,SAAS;IAKpG;;;;OAIG;IACH,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,QAAQ,EAAE;IAUjE,OAAO;IAMP,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,SAAS;IAM/D,MAAM,CAAC,CAAC,EAAE,QAAQ;IAUlB,WAAW,IAAI,QAAQ,GAAG,IAAI;IAI9B,WAAW,IAAI,YAAY;IAQ3B,QAAQ,IAAI,MAAM;CAGnB;AAiOD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,UAe1E"}