@typeberry/jam 0.4.0-fcdfbb1 → 0.4.1-0a3acb2

package/README.md

@@ -8,6 +8,11 @@ Typeberry is a TypeScript implementation of [JAM protocol](https://graypaper.com
 PRs unless the contributor waives any claims to the prize and copy rights for
 the submitted code. By creating the PR you accept this requirement.**
 
+## Links
+
+- [Documentation](https://fluffylabs.dev/typeberry)
+- [Performance Charts](https://typeberry.fluffylabs.dev)
+
 ## Implementation status
 
 Gray Paper compliance can be controlled via `GP_VERSION` environment variable.

package/bootstrap-generator.mjs

@@ -2878,7 +2878,11 @@ var TestSuite;
 })(TestSuite || (TestSuite = {}));
 const ALL_VERSIONS_IN_ORDER = [GpVersion.V0_6_7, GpVersion.V0_7_0, GpVersion.V0_7_1, GpVersion.V0_7_2];
 const DEFAULT_SUITE = TestSuite.W3F_DAVXY;
-const DEFAULT_VERSION = GpVersion.V0_7_2;
+/**
+ * Current version is set to track the jam-conformance testing.
+ * Since we are currently at 0.7.1 not 0.7.2, we set our default version accordingly.
+ */
+const DEFAULT_VERSION = GpVersion.V0_7_1;
 const env = typeof process === "undefined" ? {} : process.env;
 let CURRENT_VERSION = parseCurrentVersion(env.GP_VERSION) ?? DEFAULT_VERSION;
 let CURRENT_SUITE = parseCurrentSuite(env.TEST_SUITE) ?? DEFAULT_SUITE;
@@ -2937,8 +2941,8 @@ class Compatibility {
     /**
      * Allows selecting different values for different Gray Paper versions from one record.
      *
-     * @param fallback The default value to return if no value is found for the current.
-     * @param record A record mapping versions to values, checking if the version is greater or equal to the current version.
+     * fallback The default value to return if no value is found for the current.
+     * versions A record mapping versions to values, checking if the version is greater or equal to the current version.
      * @returns The value for the current version, or the default value.
      */
     static selectIfGreaterOrEqual({ fallback, versions, }) {
@@ -3101,7 +3105,7 @@ const workspacePathFix = dev_env.NODE_ENV === "development"
 
 ;// CONCATENATED MODULE: ./packages/core/utils/opaque.ts
 /**
- * @fileoverview `Opaque<Type, Token>` constructs a unique type which is a subset of Type with a
+ * `Opaque<Type, Token>` constructs a unique type which is a subset of Type with a
  * specified unique token Token. It means that base type cannot be assigned to unique type by accident.
  * Good examples of opaque types include:
  * - JWTs or other tokens - these are special kinds of string used for authorization purposes.
@@ -6708,9 +6712,438 @@ class ArrayView {
     }
 }
 
+;// CONCATENATED MODULE: ./packages/core/collections/blob-dictionary.ts
+
+
+/** A map which uses byte blobs as keys */
+class BlobDictionary extends WithDebug {
+    mapNodeThreshold;
+    /**
+     * The root node of the dictionary.
+     *
+     * This is the main internal data structure that organizes entries
+     * in a tree-like fashion (array-based nodes up to `mapNodeThreshold`,
+     * map-based nodes beyond it). All insertions, updates, and deletions
+     * operate through this structure.
+     */
+    root = Node.withList();
+    /**
+     * Auxiliary map that stores references to the original keys and their values.
+     *
+     * - Overriding a value in the main structure does not replace the original key reference.
+     * - Used for efficient iteration over `keys()`, `values()`, `entries()`, and computing `size`.
+     */
+    keyvals = new Map();
+    /**
+     * Protected constructor used internally by `BlobDictionary.new`
+     * and `BlobDictionary.fromEntries`.
+     *
+     * This enforces controlled instantiation — users should create instances
+     * through the provided static factory methods instead of calling the
+     * constructor directly.
+     *
+     * @param mapNodeThreshold - The threshold that determines when the dictionary
+     * switches from using an array-based (`ListChildren`) node to a map-based (`MapChildren`) node for storing entries.
+     */
+    constructor(mapNodeThreshold) {
+        super();
+        this.mapNodeThreshold = mapNodeThreshold;
+    }
+    /**
+     * Returns the number of entries in the dictionary.
+     *
+     * The count is derived from the auxiliary `keyvals` map, which stores
+     * all original key references and their associated values. This ensures
+     * that the `size` reflects the actual number of entries, independent of
+     * internal overrides in the main `root` structure.
+     *
+     * @returns The total number of entries in the dictionary.
+     */
+    get size() {
+        return this.keyvals.size;
+    }
+    [TEST_COMPARE_USING]() {
+        const vals = Array.from(this);
+        vals.sort((a, b) => a[0].compare(b[0]).value);
+        return vals;
+    }
+    /**
+     * Creates an empty `BlobDictionary`.
+     *
+     * @param mapNodeThreshold - The threshold that determines when the dictionary
+     * switches from using an array-based (`ListChildren`) node to a map-based (`MapChildren`) node for storing entries.
+     * Defaults to `0`.
+     *
+     * @returns A new, empty `BlobDictionary` instance.
+     */
+    static new(mapNodeThreshold = 0) {
+        return new BlobDictionary(mapNodeThreshold);
+    }
+    /**
+     * Creates a new `BlobDictionary` initialized with the given entries.
+     *
+     * @param entries - An array of `[key, value]` pairs used to populate the dictionary.
+     * @param mapNodeThreshold - The threshold that determines when the dictionary
+     * switches from using an array-based (`ListChildren`) node to a map-based (`MapChildren`) node for storing entries.
+     * Defaults to `0`.
+     *
+     * @returns A new `BlobDictionary` containing the provided entries.
+     */
+    static fromEntries(entries, mapNodeThreshold) {
+        const dict = BlobDictionary.new(mapNodeThreshold);
+        for (const [key, value] of entries) {
+            dict.set(key, value);
+        }
+        return dict;
+    }
+    /**
+     * Internal helper that inserts, updates or deletes an entry in the dictionary.
+     *
+     * Behaviour details:
+     * - Passing `undefined` as `value` indicates a deletion. (E.g. `delete` uses `internalSet(key, undefined)`.)
+     * - When an add (new entry) or a delete actually changes the structure, the method returns the affected leaf node.
+     * - When the call only overrides an existing value (no structural add/delete), the method returns `null`.
+     *
+     * This method is intended for internal use by the dictionary implementation and allows `undefined` as a
+     * sentinel value to signal removals.
+     *
+     * @param key - The key to insert, update or remove.
+     * @param value - The value to associate with the key, or `undefined` to remove the key.
+     * @returns The leaf node created or removed on add/delete, or `null` if the operation only overwrote an existing value.
+     */
+    internalSet(key, value) {
+        let node = this.root;
+        const keyChunkGenerator = key.chunks(CHUNK_SIZE);
+        let depth = 0;
+        for (;;) {
+            const maybeKeyChunk = keyChunkGenerator.next().value;
+            if (maybeKeyChunk === undefined) {
+                if (value === undefined) {
+                    return node.remove(key);
+                }
+                return node.set(key, value);
+            }
+            const keyChunk = opaque_asOpaqueType(maybeKeyChunk);
+            if (node.children instanceof ListChildren) {
+                const subkey = bytes_BytesBlob.blobFrom(key.raw.subarray(CHUNK_SIZE * depth));
+                const leaf = value !== undefined ? node.children.insert(subkey, { key, value }) : node.children.remove(subkey);
+                if (subkey.length > CHUNK_SIZE && node.children.children.length > this.mapNodeThreshold) {
+                    node.convertListChildrenToMap();
+                }
+                return leaf;
+            }
+            depth += 1;
+            const children = node.children;
+            if (children instanceof ListChildren) {
+                throw new Error("We handle list node earlier. If we fall through, we know it's for the `Map` case.");
+            }
+            if (children instanceof MapChildren) {
+                const maybeNode = children.getChild(keyChunk);
+                if (maybeNode !== undefined) {
+                    // simply go one level deeper
+                    node = maybeNode;
+                }
+                else {
+                    // we are trying to remove an item, but it does not exist
+                    if (value === undefined) {
+                        return null;
+                    }
+                    // no more child nodes, we insert a new one.
+                    const newNode = Node.withList();
+                    children.setChild(keyChunk, newNode);
+                    node = newNode;
+                }
+                continue;
+            }
+            debug_assertNever(children);
+        }
+    }
+    /**
+     * Adds a new entry to the dictionary or updates the value of an existing key.
+     *
+     * If an entry with the given key already exists, its value is replaced
+     * with the new one.
+     *
+     * @param key - The key to add or update in the dictionary.
+     * @param value - The value to associate with the specified key.
+     * @returns Nothing (`void`).
+     */
+    set(key, value) {
+        const leaf = this.internalSet(key, value);
+        if (leaf !== null) {
+            this.keyvals.set(leaf.key, leaf);
+        }
+    }
+    /**
+     * Retrieves the value associated with the given key from the dictionary.
+     *
+     * If the key does not exist, this method returns `undefined`.
+     *
+     * @param key - The key whose associated value should be retrieved.
+     * @returns The value associated with the specified key, or `undefined` if the key is not present.
+     */
+    get(key) {
+        let node = this.root;
+        const pathChunksGenerator = key.chunks(CHUNK_SIZE);
+        let depth = 0;
+        while (node !== undefined) {
+            const maybePathChunk = pathChunksGenerator.next().value;
+            if (node.children instanceof ListChildren) {
+                const subkey = bytes_BytesBlob.blobFrom(key.raw.subarray(depth * CHUNK_SIZE));
+                const child = node.children.find(subkey);
+                if (child !== null) {
+                    return child.value;
+                }
+            }
+            if (maybePathChunk === undefined) {
+                return node.getLeaf()?.value;
+            }
+            if (node.children instanceof MapChildren) {
+                const pathChunk = opaque_asOpaqueType(maybePathChunk);
+                node = node.children.getChild(pathChunk);
+                depth += 1;
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Checks whether the dictionary contains an entry for the given key.
+     *
+     * ⚠️ **Note:** Avoid using `has(...)` together with `get(...)` in a pattern like this:
+     *
+     * ```ts
+     * if (dict.has(key)) {
+     *   const value = dict.get(key);
+     *   ...
+     * }
+     * ```
+     *
+     * This approach performs two lookups for the same key.
+     *
+     * Instead, prefer the following pattern, which retrieves the value once:
+     *
+     * ```ts
+     * const value = dict.get(key);
+     * if (value !== undefined) {
+     *   ...
+     * }
+     * ```
+     *
+     * @param key - The key to check for.
+     * @returns `true` if the dictionary contains an entry for the given key, otherwise `false`.
+     */
+    has(key) {
+        return this.get(key) !== undefined;
+    }
+    /**
+     * Removes an entry with the specified key from the dictionary.
+     *
+     * Internally, this calls {@link internalSet} with `undefined` to mark the entry as deleted.
+     *
+     * @param key - The key of the entry to remove.
+     * @returns `true` if an entry was removed (i.e. the key existed), otherwise `false`.
+     */
+    delete(key) {
+        const leaf = this.internalSet(key, undefined);
+        if (leaf !== null) {
+            this.keyvals.delete(leaf.key);
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Returns an iterator over the keys in the dictionary.
+     *
+     * The iterator yields each key in insertion order.
+     *
+     * @returns An iterator over all keys in the dictionary.
+     */
+    keys() {
+        return this.keyvals.keys();
+    }
+    /**
+     * Returns an iterator over the values in the dictionary.
+     *
+     * The iterator yields each value in insertion order.
+     *
+     * @returns An iterator over all values in the dictionary.
+     */
+    *values() {
+        for (const leaf of this.keyvals.values()) {
+            yield leaf.value;
+        }
+    }
+    /**
+     * Returns an iterator over the `[key, value]` pairs in the dictionary.
+     *
+     * The iterator yields entries in insertion order.
+     *
+     * @returns An iterator over `[key, value]` tuples for each entry in the dictionary.
+     */
+    *entries() {
+        for (const leaf of this.keyvals.values()) {
+            yield [leaf.key, leaf.value];
+        }
+    }
+    /**
+     * Default iterator for the dictionary.
+     *
+     * Equivalent to calling {@link entries}.
+     * Enables iteration with `for...of`:
+     *
+     * ```ts
+     * for (const [key, value] of dict) {
+     *   ...
+     * }
+     * ```
+     *
+     * @returns An iterator over `[key, value]` pairs.
+     */
+    [Symbol.iterator]() {
+        return this.entries();
+    }
+    /**
+     * Creates a new sorted array of values, ordered by their corresponding keys.
+     *
+     * Iterates over all entries in the dictionary and sorts them according
+     * to the provided comparator function applied to the keys.
+     *
+     * @param comparator - A comparator function that can compare two keys.
+     *
+     * @returns A new array containing all values from the dictionary,
+     * sorted according to their keys.
+     */
+    toSortedArray(comparator) {
+        const vals = Array.from(this);
+        vals.sort((a, b) => comparator(a[0], b[0]).value);
+        return vals.map((x) => x[1]);
+    }
+}
+const CHUNK_SIZE = 6;
+/**
+ * A function to transform a bytes chunk (up to 6 bytes into U48 number)
+ *
+ * Note that it uses 3 additional bits to store length(`value * 8 + len;`),
+ * It is needed to distinguish shorter chunks that have 0s at the end, for example: [1, 2] and [1, 2, 0]
+ * */
+function bytesAsU48(bytes) {
+    const len = bytes.length;
+    debug_check `${len <= CHUNK_SIZE} Length has to be <= ${CHUNK_SIZE}, got: ${len}`;
+    let value = bytes[3] | (bytes[2] << 8) | (bytes[1] << 16) | (bytes[0] << 24);
+    for (let i = 4; i < bytes.length; i++) {
+        value = value * 256 + bytes[i];
+    }
+    return value * 8 + len;
+}
+class Node {
+    leaf;
+    children;
+    convertListChildrenToMap() {
+        if (!(this.children instanceof ListChildren)) {
+            return;
+        }
+        this.children = MapChildren.fromListNode(this.children);
+    }
+    static withList() {
+        return new Node(undefined, ListChildren.new());
+    }
+    static withMap() {
+        return new Node(undefined, MapChildren.new());
+    }
+    constructor(leaf, children) {
+        this.leaf = leaf;
+        this.children = children;
+    }
+    getLeaf() {
+        return this.leaf;
+    }
+    remove(_key) {
+        if (this.leaf === undefined) {
+            return null;
+        }
+        const removedLeaf = this.leaf;
+        this.leaf = undefined;
+        return removedLeaf;
+    }
+    set(key, value) {
+        if (this.leaf === undefined) {
+            this.leaf = { key, value };
+            return this.leaf;
+        }
+        this.leaf.value = value;
+        return null;
+    }
+}
+class ListChildren {
+    children = [];
+    constructor() { }
+    find(key) {
+        const result = this.children.find((item) => item[0].isEqualTo(key));
+        if (result !== undefined) {
+            return result[1];
+        }
+        return null;
+    }
+    remove(key) {
+        const existingIndex = this.children.findIndex((item) => item[0].isEqualTo(key));
+        if (existingIndex >= 0) {
+            const ret = this.children.splice(existingIndex, 1);
+            return ret[0][1];
+        }
+        return null;
+    }
+    insert(key, leaf) {
+        const existingIndex = this.children.findIndex((item) => item[0].isEqualTo(key));
+        if (existingIndex >= 0) {
+            const existing = this.children[existingIndex];
+            existing[1].value = leaf.value;
+            return null;
+        }
+        this.children.push([key, leaf]);
+        return leaf;
+    }
+    static new() {
+        return new ListChildren();
+    }
+}
+class MapChildren {
+    children = new Map();
+    constructor() { }
+    static new() {
+        return new MapChildren();
+    }
+    static fromListNode(node) {
+        const mapNode = new MapChildren();
+        for (const [key, leaf] of node.children) {
+            const currentKeyChunk = opaque_asOpaqueType(bytes_BytesBlob.blobFrom(key.raw.subarray(0, CHUNK_SIZE)));
+            const subKey = bytes_BytesBlob.blobFrom(key.raw.subarray(CHUNK_SIZE));
+            let child = mapNode.getChild(currentKeyChunk);
+            if (child === undefined) {
+                child = Node.withList();
+                mapNode.setChild(currentKeyChunk, child);
+            }
+            const children = child.children;
+            children.insert(subKey, leaf);
+        }
+        return mapNode;
+    }
+    getChild(keyChunk) {
+        const chunkAsNumber = bytesAsU48(keyChunk.raw);
+        return this.children.get(chunkAsNumber);
+    }
+    setChild(keyChunk, node) {
+        const chunkAsNumber = bytesAsU48(keyChunk.raw);
+        this.children.set(chunkAsNumber, node);
+    }
+}
+
 ;// CONCATENATED MODULE: ./packages/core/collections/hash-dictionary.ts
-/** A map which uses hashes as keys. */
-class HashDictionary {
+/**
+ * A map which uses hashes as keys.
+ *
+ * @deprecated
+ * */
+class StringHashDictionary {
     // TODO [ToDr] [crit] We can't use `TrieHash` directly in the map,
     // because of the way it's being compared. Hence having `string` here.
     // This has to be benchmarked and re-written to a custom map most likely.
@@ -6776,6 +7209,17 @@ class HashDictionary {
     }
 }
 
+/**
+ * A value that indicates when `BlobDictionary` transforms Array nodes into Map nodes.
+ * In practice, it doesn't matter much because, in real life, arrays in this structure usually have a length close to 1.
+ */
+const BLOB_DICTIONARY_THRESHOLD = 5;
+class HashDictionary extends BlobDictionary {
+    constructor() {
+        super(BLOB_DICTIONARY_THRESHOLD);
+    }
+}
+
 ;// CONCATENATED MODULE: ./packages/core/collections/hash-set.ts
 
 /** A set specialized for storing hashes. */
@@ -7240,6 +7684,18 @@ class SortedSet extends SortedArray {
 
 
 
+function getTruncatedKey(key) {
+    // Always return exactly TRUNCATED_HASH_SIZE bytes.
+    if (key.length === TRUNCATED_HASH_SIZE) {
+        return key;
+    }
+    return bytes_Bytes.fromBlob(key.raw.subarray(0, TRUNCATED_HASH_SIZE), TRUNCATED_HASH_SIZE);
+}
+/**
+ * A value that indicates when `BlobDictionary` transforms Array nodes into Map nodes.
+ * In practice, it doesn't matter much because, in real life, arrays in this structure usually have a length close to 1.
+ */
+const truncated_hash_dictionary_BLOB_DICTIONARY_THRESHOLD = 5;
 /**
  * A collection of hash-based keys (likely `StateKey`s) which ignores
  * differences on the last byte.
@@ -7252,48 +7708,37 @@ class TruncatedHashDictionary {
      * Each key will be copied and have the last byte replace with a 0.
      */
     static fromEntries(entries) {
-        /** Copy key bytes of an entry and replace the last one with 0. */
-        const mapped = Array.from(entries).map(([key, value]) => {
-            const newKey = bytes_Bytes.zero(HASH_SIZE).asOpaque();
-            newKey.raw.set(key.raw.subarray(0, TRUNCATED_HASH_SIZE));
-            return [newKey, value];
-        });
-        return new TruncatedHashDictionary(HashDictionary.fromEntries(mapped));
+        return new TruncatedHashDictionary(BlobDictionary.fromEntries(Array.from(entries).map(([key, value]) => [getTruncatedKey(key), value]), truncated_hash_dictionary_BLOB_DICTIONARY_THRESHOLD));
     }
-    /** A truncated key which we re-use to query the dictionary. */
-    truncatedKey = bytes_Bytes.zero(HASH_SIZE).asOpaque();
     constructor(dict) {
         this.dict = dict;
     }
     [TEST_COMPARE_USING]() {
-        return this.dict;
+        return Array.from(this.dict);
     }
     /** Return number of items in the dictionary. */
     get size() {
         return this.dict.size;
     }
     /** Retrieve a value that matches the key on `TRUNCATED_HASH_SIZE`. */
-    get(fullKey) {
-        this.truncatedKey.raw.set(fullKey.raw.subarray(0, TRUNCATED_HASH_SIZE));
-        return this.dict.get(this.truncatedKey);
+    get(key) {
+        const truncatedKey = getTruncatedKey(key);
+        return this.dict.get(truncatedKey);
     }
     /** Return true if the key is present in the dictionary */
-    has(fullKey) {
-        this.truncatedKey.raw.set(fullKey.raw.subarray(0, TRUNCATED_HASH_SIZE));
-        return this.dict.has(this.truncatedKey);
+    has(key) {
+        const truncatedKey = getTruncatedKey(key);
+        return this.dict.has(truncatedKey);
     }
     /** Set or update a value that matches the key on `TRUNCATED_HASH_SIZE`. */
-    set(fullKey, value) {
-        // NOTE we can't use the the shared key here, since the collection will
-        // store the key for us, hence the copy.
-        const key = bytes_Bytes.zero(HASH_SIZE);
-        key.raw.set(fullKey.raw.subarray(0, TRUNCATED_HASH_SIZE));
-        this.dict.set(key.asOpaque(), value);
+    set(key, value) {
+        const truncatedKey = getTruncatedKey(key);
+        this.dict.set(truncatedKey, value);
     }
     /** Remove a value that matches the key on `TRUNCATED_HASH_SIZE`. */
-    delete(fullKey) {
-        this.truncatedKey.raw.set(fullKey.raw.subarray(0, TRUNCATED_HASH_SIZE));
-        this.dict.delete(this.truncatedKey);
+    delete(key) {
+        const truncatedKey = getTruncatedKey(key);
+        this.dict.delete(truncatedKey);
     }
     /** Iterator over values of the dictionary. */
     values() {
@@ -7301,9 +7746,7 @@ class TruncatedHashDictionary {
     }
     /** Iterator over entries of the dictionary (with truncated keys) */
     *entries() {
-        for (const [key, value] of this.dict.entries()) {
-            yield [bytes_Bytes.fromBlob(key.raw.subarray(0, TRUNCATED_HASH_SIZE), TRUNCATED_HASH_SIZE).asOpaque(), value];
-        }
+        yield* this.dict.entries();
     }
     [Symbol.iterator]() {
         return this.entries();
@@ -7320,6 +7763,7 @@ class TruncatedHashDictionary {
 
 
 
+
 ;// CONCATENATED MODULE: ./packages/jam/config/chain-spec.ts
 
 
@@ -10200,11 +10644,32 @@ const ENTROPY_ENTRIES = 4;
 
 var UpdatePreimageKind;
 (function (UpdatePreimageKind) {
-    /** Insert new preimage and optionally update it's lookup history. */
+    /**
+     * Insert new preimage and optionally update it's lookup history.
+     *
+     * Used in: `provide`
+     *
+     * https://graypaper.fluffylabs.dev/#/ab2cdbd/383904383904?v=0.7.2
+     */
     UpdatePreimageKind[UpdatePreimageKind["Provide"] = 0] = "Provide";
-    /** Remove a preimage and it's lookup history. */
+    /**
+     * Remove a preimage and it's lookup history.
+     *
+     * Used in: `forget` and `eject`
+     *
+     * https://graypaper.fluffylabs.dev/#/ab2cdbd/38c701380202?v=0.7.2
+     * https://graypaper.fluffylabs.dev/#/ab2cdbd/379102379302?v=0.7.2
+     */
     UpdatePreimageKind[UpdatePreimageKind["Remove"] = 1] = "Remove";
-    /** update or add lookup history for preimage hash/len to given value. */
+    /**
+     * Update or add lookup history for preimage hash/len to given value.
+     *
+     * Used in: `solicit` and `forget`
+     *
+     * https://graypaper.fluffylabs.dev/#/ab2cdbd/382802382802?v=0.7.2
+     * https://graypaper.fluffylabs.dev/#/ab2cdbd/384002384b02?v=0.7.2
+     * https://graypaper.fluffylabs.dev/#/ab2cdbd/38c60038ea00?v=0.7.2
+     */
     UpdatePreimageKind[UpdatePreimageKind["UpdateOrAdd"] = 2] = "UpdateOrAdd";
 })(UpdatePreimageKind || (UpdatePreimageKind = {}));
 /**
@@ -10212,7 +10677,7 @@ var UpdatePreimageKind;
  *
  * Can be one of the following cases:
  * 1. Provide a new preimage blob and set the lookup history to available at `slot`.
- * 2. Remove (expunge) a preimage and it's lookup history.
+ * 2. Remove (forget) a preimage and it's lookup history.
  * 3. Update `LookupHistory` with given value.
  */
 class state_update_UpdatePreimage {