@acala-network/chopsticks-core 1.2.1 → 1.2.2

package/dist/cjs/blockchain/block.d.ts

@@ -48,6 +48,10 @@ export declare class Block {
      * Get the block storage by key.
      */
     get(key: string): Promise<HexString | undefined>;
+    /**
+     * Get the block storage by key.
+     */
+    getMany(keys: string[]): Promise<Array<HexString | undefined>>;
     read<T extends string>(type: T, query: StorageEntry, ...args: any[]): Promise<import("@polkadot/types/types").DetectCodec<import("@polkadot/types-codec/types").Codec, T> | undefined>;
     /**
      * Get paged storage keys.

package/dist/cjs/blockchain/block.js

@@ -127,6 +127,19 @@ class Block {
                 return val;
         }
     }
+    /**
+   * Get the block storage by key.
+   */ async getMany(keys) {
+        const vals = await this.storage.getMany(keys, true);
+        return vals.map((val)=>{
+            switch(val){
+                case _storagelayer.StorageValueKind.Deleted:
+                    return undefined;
+                default:
+                    return val;
+            }
+        });
+    }
     async read(type, query, ...args) {
         const key = (0, _index.compactHex)(query(...args));
         const value = await this.get(key);

package/dist/cjs/blockchain/inherent/parachain/validation-data.js

@@ -12,6 +12,7 @@ const _types = require("@polkadot/types");
 const _util = require("@polkadot/util");
 const _utilcrypto = require("@polkadot/util-crypto");
 const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
+const _logger = require("../../../logger.js");
 const _index = require("../../../utils/index.js");
 const _proof = require("../../../utils/proof.js");
 const _index1 = require("../../../wasm-executor/index.js");
@@ -20,6 +21,9 @@ function _interop_require_default(obj) {
         default: obj
     };
 }
+const logger = _logger.defaultLogger.child({
+    name: 'parachain-validation-data'
+});
 const MOCK_VALIDATION_DATA = {
     validationData: {
         relayParentNumber: 1000,
@@ -46,7 +50,7 @@ const MOCK_VALIDATION_DATA = {
     horizontalMessages: [],
     downwardMessages: []
 };
-const getValidationData = async (parent)=>{
+const getValidationData = async (parent, fallback = true)=>{
     const meta = await parent.meta;
     if (parent.number === 0) {
         const { trieRootHash, nodes } = await (0, _index1.createProof)(MOCK_VALIDATION_DATA.relayChainState.trieNodes, []);
@@ -61,15 +65,39 @@ const getValidationData = async (parent)=>{
             }
         };
     }
-    const extrinsics = await parent.extrinsics;
-    const validationDataExtrinsic = extrinsics.find((extrinsic)=>{
-        const firstArg = meta.registry.createType('GenericExtrinsic', extrinsic)?.args?.[0];
-        return firstArg && 'validationData' in firstArg;
-    });
-    if (!validationDataExtrinsic) {
-        throw new Error('Missing validation data from block');
+    try {
+        const extrinsics = await parent.extrinsics;
+        const validationDataExtrinsic = extrinsics.find((extrinsic)=>{
+            const firstArg = meta.registry.createType('GenericExtrinsic', extrinsic)?.args?.[0];
+            return firstArg && 'validationData' in firstArg;
+        });
+        if (!validationDataExtrinsic) {
+            throw new Error('Missing validation data from block');
+        }
+        return meta.registry.createType('GenericExtrinsic', validationDataExtrinsic).args[0].toJSON();
+    } catch (e) {
+        logger.warn('Failed to get validation data from block %d %s', parent.number, e);
+        if (fallback) {
+            // this could fail due to wasm override that breaks the validation data format
+            // so we will try parent's parent
+            const grandParent = await parent.parentBlock;
+            if (grandParent) {
+                const data = await getValidationData(grandParent, false);
+                return {
+                    ...data,
+                    validationData: {
+                        ...data.validationData,
+                        relayParentNumber: data.validationData.relayParentNumber + 2
+                    }
+                };
+            } else {
+                throw e;
+            }
+        } else {
+            // fallback failed, throw error
+            throw e;
+        }
     }
-    return meta.registry.createType('GenericExtrinsic', validationDataExtrinsic).args[0].toJSON();
 };
 class SetValidationData {
     async createInherents(newBlock, params) {
@@ -97,7 +125,12 @@ class SetValidationData {
             if (key === _proof.WELL_KNOWN_KEYS.CURRENT_SLOT) {
                 // increment current slot
                 const relayCurrentSlot = decoded[key] ? meta.registry.createType('Slot', (0, _util.hexToU8a)(decoded[key])).toNumber() : await (0, _index.getCurrentSlot)(parent) * relaySlotIncrease;
-                const newSlot = meta.registry.createType('Slot', relayCurrentSlot + relaySlotIncrease);
+                const newSlot = meta.registry.createType('Slot', relayCurrentSlot + relaySlotIncrease + 1) // +1 to be safe
+                ;
+                logger.debug({
+                    relayCurrentSlot,
+                    newSlot: newSlot.toNumber()
+                }, 'Updating relay current slot');
                 newEntries.push([
                     key,
                     (0, _util.u8aToHex)(newSlot.toU8a())

package/dist/cjs/blockchain/storage-layer.d.ts

@@ -7,10 +7,18 @@ export declare enum StorageValueKind {
 }
 export type StorageValue = string | StorageValueKind | undefined;
 export interface StorageLayerProvider {
+    /**
+     * Returns true if key is deleted
+     */
+    deleted(key: string): boolean;
     /**
      * Get the value of a storage key.
      */
     get(key: string, cache: boolean): Promise<StorageValue>;
+    /**
+     * Get the value of many storage keys.
+     */
+    getMany(keys: string[], _cache: boolean): Promise<StorageValue[]>;
     /**
      * Get paged storage keys.
      */
@@ -23,14 +31,18 @@ export interface StorageLayerProvider {
 export declare class RemoteStorageLayer implements StorageLayerProvider {
     #private;
     constructor(api: Api, at: HexString, db: Database | undefined);
+    deleted(_key: string): boolean;
     get(key: string, _cache: boolean): Promise<StorageValue>;
+    getMany(keys: string[], _cache: boolean): Promise<StorageValue[]>;
     findNextKey(prefix: string, startKey: string, _knownBest?: string): Promise<string | undefined>;
     getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise<string[]>;
 }
 export declare class StorageLayer implements StorageLayerProvider {
     #private;
     constructor(parent?: StorageLayerProvider);
+    deleted(key: string): boolean;
     get(key: string, cache: boolean): Promise<StorageValue | undefined>;
+    getMany(keys: string[], cache: boolean): Promise<StorageValue[]>;
     set(key: string, value: StorageValue): void;
     setAll(values: Record<string, StorageValue | null> | [string, StorageValue | null][]): void;
     findNextKey(prefix: string, startKey: string, knownBest?: string): Promise<string | undefined>;

package/dist/cjs/blockchain/storage-layer.js

@@ -89,6 +89,9 @@ var StorageValueKind = /*#__PURE__*/ function(StorageValueKind) {
 }({});
 var _api = /*#__PURE__*/ new WeakMap(), _at = /*#__PURE__*/ new WeakMap(), _db = /*#__PURE__*/ new WeakMap(), _keyCache = /*#__PURE__*/ new WeakMap(), _defaultChildKeyCache = /*#__PURE__*/ new WeakMap();
 class RemoteStorageLayer {
+    deleted(_key) {
+        return false;
+    }
     async get(key, _cache) {
         if (_class_private_field_get(this, _db)) {
             const res = await _class_private_field_get(this, _db).queryStorage(_class_private_field_get(this, _at), key);
@@ -104,6 +107,48 @@ class RemoteStorageLayer {
         _class_private_field_get(this, _db)?.saveStorage(_class_private_field_get(this, _at), key, data);
         return data ?? undefined;
     }
+    async getMany(keys, _cache) {
+        const result = [];
+        let pending = keys.map((key, idx)=>({
+                key,
+                idx
+            }));
+        if (_class_private_field_get(this, _db)) {
+            const results = await Promise.all(pending.map(({ key })=>_class_private_field_get(this, _db).queryStorage(_class_private_field_get(this, _at), key)));
+            const oldPending = pending;
+            pending = [];
+            results.forEach((res, idx)=>{
+                if (res) {
+                    result[idx] = res.value ?? undefined;
+                } else {
+                    pending.push({
+                        key: oldPending[idx].key,
+                        idx
+                    });
+                }
+            });
+        }
+        if (pending.length) {
+            logger.trace({
+                at: _class_private_field_get(this, _at),
+                keys
+            }, 'RemoteStorageLayer getMany');
+            const data = await _class_private_field_get(this, _api).getStorageBatch('0x', pending.map(({ key })=>key), _class_private_field_get(this, _at));
+            data.forEach(([, res], idx)=>{
+                result[pending[idx].idx] = res ?? undefined;
+            });
+            if (_class_private_field_get(this, _db)?.saveStorageBatch) {
+                _class_private_field_get(this, _db)?.saveStorageBatch(data.map(([key, value])=>({
+                        key,
+                        value,
+                        blockHash: _class_private_field_get(this, _at)
+                    })));
+            } else if (_class_private_field_get(this, _db)) {
+                data.forEach(([key, value])=>_class_private_field_get(this, _db)?.saveStorage(_class_private_field_get(this, _at), key, value));
+            }
+        }
+        return result;
+    }
     async findNextKey(prefix, startKey, _knownBest) {
         const keys = await this.getKeysPaged(prefix, 1, startKey);
         return keys[0];
@@ -205,6 +250,18 @@ class RemoteStorageLayer {
 }
 var _store = /*#__PURE__*/ new WeakMap(), _keys = /*#__PURE__*/ new WeakMap(), _deletedPrefix = /*#__PURE__*/ new WeakMap(), _parent = /*#__PURE__*/ new WeakMap(), _addKey = /*#__PURE__*/ new WeakSet(), _removeKey = /*#__PURE__*/ new WeakSet();
 class StorageLayer {
+    deleted(key) {
+        if (_class_private_field_get(this, _store).has(key)) {
+            return _class_private_field_get(this, _store).get(key) === "Deleted";
+        }
+        if (_class_private_field_get(this, _deletedPrefix).some((dp)=>key.startsWith(dp))) {
+            return true;
+        }
+        if (_class_private_field_get(this, _parent)) {
+            return _class_private_field_get(this, _parent).deleted(key);
+        }
+        return false;
+    }
     async get(key, cache) {
         if (_class_private_field_get(this, _store).has(key)) {
             return _class_private_field_get(this, _store).get(key);
@@ -221,6 +278,33 @@ class StorageLayer {
         }
         return undefined;
     }
+    async getMany(keys, cache) {
+        const result = [];
+        const pending = [];
+        const preloadedPromises = keys.map(async (key, idx)=>{
+            if (_class_private_field_get(this, _store).has(key)) {
+                result[idx] = await _class_private_field_get(this, _store).get(key);
+            } else if (_class_private_field_get(this, _deletedPrefix).some((dp)=>key.startsWith(dp))) {
+                result[idx] = "Deleted";
+            } else {
+                pending.push({
+                    key,
+                    idx
+                });
+            }
+        });
+        if (pending.length && _class_private_field_get(this, _parent)) {
+            const vals = await _class_private_field_get(this, _parent).getMany(pending.map((p)=>p.key), false);
+            vals.forEach((val, idx)=>{
+                if (cache) {
+                    _class_private_field_get(this, _store).set(pending[idx].key, val);
+                }
+                result[pending[idx].idx] = val;
+            });
+        }
+        await Promise.all(preloadedPromises);
+        return result;
+    }
     set(key, value) {
         switch(value){
             case "Deleted":
@@ -283,7 +367,7 @@ class StorageLayer {
             const next = await this.findNextKey(prefix, startKey, undefined);
             if (!next) break;
             startKey = next;
-            if (await this.get(next, false) === "Deleted") continue;
+            if (this.deleted(next)) continue;
             keys.push(next);
         }
         return keys;

package/dist/cjs/rpc/rpc-spec/storage-common.js

@@ -24,10 +24,10 @@ async function getDescendantValues(block, params) {
         ...params,
         pageSize: PAGE_SIZE
     });
-    const items = await Promise.all(keys.map((key)=>block.get(key).then((value)=>({
-                key,
-                value
-            }))));
+    const items = (await block.getMany(keys)).map((value, idx)=>({
+            key: keys[idx],
+            value
+        }));
     if (keys.length < PAGE_SIZE) {
         return {
             items,

package/dist/esm/blockchain/block.d.ts

@@ -48,6 +48,10 @@ export declare class Block {
      * Get the block storage by key.
      */
     get(key: string): Promise<HexString | undefined>;
+    /**
+     * Get the block storage by key.
+     */
+    getMany(keys: string[]): Promise<Array<HexString | undefined>>;
     read<T extends string>(type: T, query: StorageEntry, ...args: any[]): Promise<import("@polkadot/types/types").DetectCodec<import("@polkadot/types-codec/types").Codec, T> | undefined>;
     /**
      * Get paged storage keys.

package/dist/esm/blockchain/block.js

@@ -117,6 +117,19 @@ import { RemoteStorageLayer, StorageLayer, StorageValueKind } from './storage-la
                 return val;
         }
     }
+    /**
+   * Get the block storage by key.
+   */ async getMany(keys) {
+        const vals = await this.storage.getMany(keys, true);
+        return vals.map((val)=>{
+            switch(val){
+                case StorageValueKind.Deleted:
+                    return undefined;
+                default:
+                    return val;
+            }
+        });
+    }
     async read(type, query, ...args) {
         const key = compactHex(query(...args));
         const value = await this.get(key);

package/dist/esm/blockchain/inherent/parachain/validation-data.js

@@ -2,9 +2,13 @@ import { GenericExtrinsic } from '@polkadot/types';
 import { hexToU8a, u8aConcat, u8aToHex } from '@polkadot/util';
 import { blake2AsHex, blake2AsU8a } from '@polkadot/util-crypto';
 import _ from 'lodash';
+import { defaultLogger } from '../../../logger.js';
 import { compactHex, getCurrentSlot, getParaId } from '../../../utils/index.js';
 import { dmqMqcHead, hrmpChannels, hrmpEgressChannelIndex, hrmpIngressChannelIndex, paraHead, upgradeGoAheadSignal, WELL_KNOWN_KEYS } from '../../../utils/proof.js';
 import { createProof, decodeProof } from '../../../wasm-executor/index.js';
+const logger = defaultLogger.child({
+    name: 'parachain-validation-data'
+});
 const MOCK_VALIDATION_DATA = {
     validationData: {
         relayParentNumber: 1000,
@@ -31,7 +35,7 @@ const MOCK_VALIDATION_DATA = {
     horizontalMessages: [],
     downwardMessages: []
 };
-const getValidationData = async (parent)=>{
+const getValidationData = async (parent, fallback = true)=>{
     const meta = await parent.meta;
     if (parent.number === 0) {
         const { trieRootHash, nodes } = await createProof(MOCK_VALIDATION_DATA.relayChainState.trieNodes, []);
@@ -46,15 +50,39 @@ const getValidationData = async (parent)=>{
             }
         };
     }
-    const extrinsics = await parent.extrinsics;
-    const validationDataExtrinsic = extrinsics.find((extrinsic)=>{
-        const firstArg = meta.registry.createType('GenericExtrinsic', extrinsic)?.args?.[0];
-        return firstArg && 'validationData' in firstArg;
-    });
-    if (!validationDataExtrinsic) {
-        throw new Error('Missing validation data from block');
+    try {
+        const extrinsics = await parent.extrinsics;
+        const validationDataExtrinsic = extrinsics.find((extrinsic)=>{
+            const firstArg = meta.registry.createType('GenericExtrinsic', extrinsic)?.args?.[0];
+            return firstArg && 'validationData' in firstArg;
+        });
+        if (!validationDataExtrinsic) {
+            throw new Error('Missing validation data from block');
+        }
+        return meta.registry.createType('GenericExtrinsic', validationDataExtrinsic).args[0].toJSON();
+    } catch (e) {
+        logger.warn('Failed to get validation data from block %d %s', parent.number, e);
+        if (fallback) {
+            // this could fail due to wasm override that breaks the validation data format
+            // so we will try parent's parent
+            const grandParent = await parent.parentBlock;
+            if (grandParent) {
+                const data = await getValidationData(grandParent, false);
+                return {
+                    ...data,
+                    validationData: {
+                        ...data.validationData,
+                        relayParentNumber: data.validationData.relayParentNumber + 2
+                    }
+                };
+            } else {
+                throw e;
+            }
+        } else {
+            // fallback failed, throw error
+            throw e;
+        }
     }
-    return meta.registry.createType('GenericExtrinsic', validationDataExtrinsic).args[0].toJSON();
 };
 export class SetValidationData {
     async createInherents(newBlock, params) {
@@ -82,7 +110,12 @@ export class SetValidationData {
             if (key === WELL_KNOWN_KEYS.CURRENT_SLOT) {
                 // increment current slot
                 const relayCurrentSlot = decoded[key] ? meta.registry.createType('Slot', hexToU8a(decoded[key])).toNumber() : await getCurrentSlot(parent) * relaySlotIncrease;
-                const newSlot = meta.registry.createType('Slot', relayCurrentSlot + relaySlotIncrease);
+                const newSlot = meta.registry.createType('Slot', relayCurrentSlot + relaySlotIncrease + 1) // +1 to be safe
+                ;
+                logger.debug({
+                    relayCurrentSlot,
+                    newSlot: newSlot.toNumber()
+                }, 'Updating relay current slot');
                 newEntries.push([
                     key,
                     u8aToHex(newSlot.toU8a())

package/dist/esm/blockchain/storage-layer.d.ts

@@ -7,10 +7,18 @@ export declare enum StorageValueKind {
 }
 export type StorageValue = string | StorageValueKind | undefined;
 export interface StorageLayerProvider {
+    /**
+     * Returns true if key is deleted
+     */
+    deleted(key: string): boolean;
     /**
      * Get the value of a storage key.
      */
     get(key: string, cache: boolean): Promise<StorageValue>;
+    /**
+     * Get the value of many storage keys.
+     */
+    getMany(keys: string[], _cache: boolean): Promise<StorageValue[]>;
     /**
      * Get paged storage keys.
      */
@@ -23,14 +31,18 @@ export interface StorageLayerProvider {
 export declare class RemoteStorageLayer implements StorageLayerProvider {
     #private;
     constructor(api: Api, at: HexString, db: Database | undefined);
+    deleted(_key: string): boolean;
     get(key: string, _cache: boolean): Promise<StorageValue>;
+    getMany(keys: string[], _cache: boolean): Promise<StorageValue[]>;
     findNextKey(prefix: string, startKey: string, _knownBest?: string): Promise<string | undefined>;
     getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise<string[]>;
 }
 export declare class StorageLayer implements StorageLayerProvider {
     #private;
     constructor(parent?: StorageLayerProvider);
+    deleted(key: string): boolean;
     get(key: string, cache: boolean): Promise<StorageValue | undefined>;
+    getMany(keys: string[], cache: boolean): Promise<StorageValue[]>;
     set(key: string, value: StorageValue): void;
     setAll(values: Record<string, StorageValue | null> | [string, StorageValue | null][]): void;
     findNextKey(prefix: string, startKey: string, knownBest?: string): Promise<string | undefined>;

package/dist/esm/blockchain/storage-layer.js

@@ -22,6 +22,9 @@ export class RemoteStorageLayer {
         this.#at = at;
         this.#db = db;
     }
+    deleted(_key) {
+        return false;
+    }
     async get(key, _cache) {
         if (this.#db) {
             const res = await this.#db.queryStorage(this.#at, key);
@@ -37,6 +40,48 @@ export class RemoteStorageLayer {
         this.#db?.saveStorage(this.#at, key, data);
         return data ?? undefined;
     }
+    async getMany(keys, _cache) {
+        const result = [];
+        let pending = keys.map((key, idx)=>({
+                key,
+                idx
+            }));
+        if (this.#db) {
+            const results = await Promise.all(pending.map(({ key })=>this.#db.queryStorage(this.#at, key)));
+            const oldPending = pending;
+            pending = [];
+            results.forEach((res, idx)=>{
+                if (res) {
+                    result[idx] = res.value ?? undefined;
+                } else {
+                    pending.push({
+                        key: oldPending[idx].key,
+                        idx
+                    });
+                }
+            });
+        }
+        if (pending.length) {
+            logger.trace({
+                at: this.#at,
+                keys
+            }, 'RemoteStorageLayer getMany');
+            const data = await this.#api.getStorageBatch('0x', pending.map(({ key })=>key), this.#at);
+            data.forEach(([, res], idx)=>{
+                result[pending[idx].idx] = res ?? undefined;
+            });
+            if (this.#db?.saveStorageBatch) {
+                this.#db?.saveStorageBatch(data.map(([key, value])=>({
+                        key,
+                        value,
+                        blockHash: this.#at
+                    })));
+            } else if (this.#db) {
+                data.forEach(([key, value])=>this.#db?.saveStorage(this.#at, key, value));
+            }
+        }
+        return result;
+    }
     async findNextKey(prefix, startKey, _knownBest) {
         const keys = await this.getKeysPaged(prefix, 1, startKey);
         return keys[0];
@@ -134,6 +179,18 @@ export class StorageLayer {
             this.#keys.splice(idx, 1);
         }
     }
+    deleted(key) {
+        if (this.#store.has(key)) {
+            return this.#store.get(key) === "Deleted";
+        }
+        if (this.#deletedPrefix.some((dp)=>key.startsWith(dp))) {
+            return true;
+        }
+        if (this.#parent) {
+            return this.#parent.deleted(key);
+        }
+        return false;
+    }
     async get(key, cache) {
         if (this.#store.has(key)) {
             return this.#store.get(key);
@@ -150,6 +207,33 @@ export class StorageLayer {
         }
         return undefined;
     }
+    async getMany(keys, cache) {
+        const result = [];
+        const pending = [];
+        const preloadedPromises = keys.map(async (key, idx)=>{
+            if (this.#store.has(key)) {
+                result[idx] = await this.#store.get(key);
+            } else if (this.#deletedPrefix.some((dp)=>key.startsWith(dp))) {
+                result[idx] = "Deleted";
+            } else {
+                pending.push({
+                    key,
+                    idx
+                });
+            }
+        });
+        if (pending.length && this.#parent) {
+            const vals = await this.#parent.getMany(pending.map((p)=>p.key), false);
+            vals.forEach((val, idx)=>{
+                if (cache) {
+                    this.#store.set(pending[idx].key, val);
+                }
+                result[pending[idx].idx] = val;
+            });
+        }
+        await Promise.all(preloadedPromises);
+        return result;
+    }
     set(key, value) {
         switch(value){
             case "Deleted":
@@ -212,7 +296,7 @@ export class StorageLayer {
             const next = await this.findNextKey(prefix, startKey, undefined);
             if (!next) break;
             startKey = next;
-            if (await this.get(next, false) === "Deleted") continue;
+            if (this.deleted(next)) continue;
             keys.push(next);
         }
         return keys;

package/dist/esm/rpc/rpc-spec/storage-common.js

@@ -3,10 +3,10 @@ export async function getDescendantValues(block, params) {
         ...params,
         pageSize: PAGE_SIZE
     });
-    const items = await Promise.all(keys.map((key)=>block.get(key).then((value)=>({
-                key,
-                value
-            }))));
+    const items = (await block.getMany(keys)).map((value, idx)=>({
+            key: keys[idx],
+            value
+        }));
     if (keys.length < PAGE_SIZE) {
         return {
             items,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@acala-network/chopsticks-core",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "author": "Acala Developers <hello@acala.network>",
   "license": "Apache-2.0",
   "type": "module",
@@ -14,7 +14,7 @@
     "depcheck": "npx depcheck"
   },
   "dependencies": {
-    "@acala-network/chopsticks-executor": "1.2.1",
+    "@acala-network/chopsticks-executor": "1.2.2",
     "@polkadot/rpc-provider": "^16.4.1",
     "@polkadot/types": "^16.4.1",
     "@polkadot/types-codec": "^16.4.1",