@acala-network/chopsticks-core 0.9.6 → 0.9.8-1

package/dist/cjs/api.d.ts

@@ -50,6 +50,7 @@ export declare class Api {
     getBlock(hash?: string): Promise<SignedBlock | null>;
     getStorage(key: string, hash?: string): Promise<`0x${string}` | null>;
     getKeysPaged(prefix: string, pageSize: number, startKey: string, hash?: string): Promise<string[]>;
+    getStorageBatch(prefix: HexString, keys: HexString[], hash?: HexString): Promise<[`0x${string}`, `0x${string}` | null][]>;
     subscribeRemoteNewHeads(cb: ProviderInterfaceCallback): Promise<string | number>;
     subscribeRemoteFinalizedHeads(cb: ProviderInterfaceCallback): Promise<string | number>;
 }

package/dist/cjs/api.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "Api", {
     }
 });
 const _index = require("./utils/index.js");
+const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
 function _check_private_redeclaration(obj, privateCollection) {
     if (privateCollection.has(obj)) {
         throw new TypeError("Cannot initialize the same private elements twice on an object");
@@ -62,6 +63,11 @@ function _define_property(obj, key, value) {
     }
     return obj;
 }
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
 var _provider = /*#__PURE__*/ new WeakMap(), _ready = /*#__PURE__*/ new WeakMap(), _chain = /*#__PURE__*/ new WeakMap(), _chainProperties = /*#__PURE__*/ new WeakMap();
 class Api {
     async disconnect() {
@@ -163,6 +169,26 @@ class Api {
             return _class_private_field_get(this, _provider).send('state_getKeysPaged', params, !!hash);
         }
     }
+    async getStorageBatch(prefix, keys, hash) {
+        const [child] = (0, _index.splitChildKey)(prefix);
+        if (child) {
+            // child storage key, use childstate_getStorageEntries
+            // strip child prefix from keys
+            const params = [
+                child,
+                keys.map((key)=>(0, _index.stripChildPrefix)(key))
+            ];
+            if (hash) params.push(hash);
+            return _class_private_field_get(this, _provider).send('childstate_getStorageEntries', params, !!hash).then((values)=>_lodash.default.zip(keys, values));
+        } else {
+            // main storage key, use state_getStorageAt
+            const params = [
+                keys
+            ];
+            if (hash) params.push(hash);
+            return _class_private_field_get(this, _provider).send('state_queryStorageAt', params, !!hash).then((result)=>result[0]?.['changes'] || []);
+        }
+    }
     async subscribeRemoteNewHeads(cb) {
         if (!_class_private_field_get(this, _provider).hasSubscriptions) {
             throw new Error('subscribeRemoteNewHeads only works with subscriptions');

package/dist/cjs/blockchain/block.js

@@ -139,12 +139,10 @@ class Block {
     /**
    * Get paged storage keys.
    */ async getKeysPaged(options) {
-        const layer = new _storagelayer.StorageLayer(this.storage);
-        await layer.fold();
         const prefix = options.prefix ?? '0x';
         const startKey = options.startKey ?? '0x';
         const pageSize = options.pageSize;
-        return layer.getKeysPaged(prefix, pageSize, startKey);
+        return this.storage.getKeysPaged(prefix, pageSize, startKey);
     }
     /**
    * Push a layer to the storage stack.

package/dist/cjs/blockchain/storage-layer.js

@@ -20,8 +20,9 @@ _export(exports, {
     }
 });
 const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
+const _index = require("../utils/index.js");
 const _logger = require("../logger.js");
-const _keycache = /*#__PURE__*/ _interop_require_wildcard(require("../utils/key-cache.js"));
+const _keycache = /*#__PURE__*/ _interop_require_default(require("../utils/key-cache.js"));
 function _check_private_redeclaration(obj, privateCollection) {
     if (privateCollection.has(obj)) {
         throw new TypeError("Cannot initialize the same private elements twice on an object");
@@ -77,47 +78,6 @@ function _interop_require_default(obj) {
         default: obj
     };
 }
-function _getRequireWildcardCache(nodeInterop) {
-    if (typeof WeakMap !== "function") return null;
-    var cacheBabelInterop = new WeakMap();
-    var cacheNodeInterop = new WeakMap();
-    return (_getRequireWildcardCache = function(nodeInterop) {
-        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
-    })(nodeInterop);
-}
-function _interop_require_wildcard(obj, nodeInterop) {
-    if (!nodeInterop && obj && obj.__esModule) {
-        return obj;
-    }
-    if (obj === null || typeof obj !== "object" && typeof obj !== "function") {
-        return {
-            default: obj
-        };
-    }
-    var cache = _getRequireWildcardCache(nodeInterop);
-    if (cache && cache.has(obj)) {
-        return cache.get(obj);
-    }
-    var newObj = {
-        __proto__: null
-    };
-    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
-    for(var key in obj){
-        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
-            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
-            if (desc && (desc.get || desc.set)) {
-                Object.defineProperty(newObj, key, desc);
-            } else {
-                newObj[key] = obj[key];
-            }
-        }
-    }
-    newObj.default = obj;
-    if (cache) {
-        cache.set(obj, newObj);
-    }
-    return newObj;
-}
 const logger = _logger.defaultLogger.child({
     name: 'layer'
 });
@@ -127,7 +87,7 @@ var StorageValueKind;
     StorageValueKind["Deleted"] = "Deleted";
     StorageValueKind["DeletedPrefix"] = "DeletedPrefix";
 })(StorageValueKind || (StorageValueKind = {}));
-var _api = /*#__PURE__*/ new WeakMap(), _at = /*#__PURE__*/ new WeakMap(), _db = /*#__PURE__*/ new WeakMap(), _keyCache = /*#__PURE__*/ new WeakMap();
+var _api = /*#__PURE__*/ new WeakMap(), _at = /*#__PURE__*/ new WeakMap(), _db = /*#__PURE__*/ new WeakMap(), _keyCache = /*#__PURE__*/ new WeakMap(), _defaultChildKeyCache = /*#__PURE__*/ new WeakMap();
 class RemoteStorageLayer {
     async get(key, _cache) {
         if (_class_private_field_get(this, _db)) {
@@ -156,14 +116,16 @@ class RemoteStorageLayer {
             pageSize,
             startKey
         }, 'RemoteStorageLayer getKeysPaged');
+        const isChild = (0, _index.isPrefixedChildKey)(prefix);
+        const minPrefixLen = isChild ? _index.CHILD_PREFIX_LENGTH : _index.PREFIX_LENGTH;
         // can't handle keyCache without prefix
-        if (prefix.length < _keycache.PREFIX_LENGTH || startKey.length < _keycache.PREFIX_LENGTH) {
+        if (prefix.length < minPrefixLen || startKey.length < minPrefixLen) {
             return _class_private_field_get(this, _api).getKeysPaged(prefix, pageSize, startKey, _class_private_field_get(this, _at));
         }
         let batchComplete = false;
         const keysPaged = [];
         while(keysPaged.length < pageSize){
-            const nextKey = await _class_private_field_get(this, _keyCache).next(startKey);
+            const nextKey = isChild ? await _class_private_field_get(this, _defaultChildKeyCache).next(startKey) : await _class_private_field_get(this, _keyCache).next(startKey);
             if (nextKey) {
                 keysPaged.push(nextKey);
                 startKey = nextKey;
@@ -178,14 +140,29 @@ class RemoteStorageLayer {
             batchComplete = batch.length < BATCH_SIZE;
             // feed the key cache
             if (batch.length > 0) {
-                _class_private_field_get(this, _keyCache).feed([
-                    startKey,
-                    ...batch
-                ]);
+                if (isChild) {
+                    _class_private_field_get(this, _defaultChildKeyCache).feed([
+                        startKey,
+                        ...batch
+                    ]);
+                } else {
+                    _class_private_field_get(this, _keyCache).feed([
+                        startKey,
+                        ...batch
+                    ]);
+                }
             }
             if (batch.length === 0) {
                 break;
             }
+            if (_class_private_field_get(this, _db)) {
+                // batch fetch storage values and save to db, they may be used later
+                _class_private_field_get(this, _api).getStorageBatch(prefix, batch, _class_private_field_get(this, _at)).then((storage)=>{
+                    for (const [key, value] of storage){
+                        _class_private_field_get(this, _db).saveStorage(_class_private_field_get(this, _at), key, value);
+                    }
+                });
+            }
         }
         return keysPaged;
     }
@@ -204,7 +181,11 @@ class RemoteStorageLayer {
         });
         _class_private_field_init(this, _keyCache, {
             writable: true,
-            value: new _keycache.default()
+            value: new _keycache.default(_index.PREFIX_LENGTH)
+        });
+        _class_private_field_init(this, _defaultChildKeyCache, {
+            writable: true,
+            value: new _keycache.default(_index.CHILD_PREFIX_LENGTH)
         });
         _class_private_field_set(this, _api, api);
         _class_private_field_set(this, _at, at);
@@ -278,30 +259,105 @@ class StorageLayer {
         }
     }
     async getKeysPaged(prefix, pageSize, startKey) {
-        if (!_class_private_field_get(this, _deletedPrefix).some((dp)=>startKey.startsWith(dp))) {
-            const remote = await _class_private_field_get(this, _parent)?.getKeysPaged(prefix, pageSize, startKey) ?? [];
-            for (const key of remote){
-                if (_class_private_field_get(this, _store).get(key) === "Deleted") {
-                    continue;
+        let parentFetchComplete = false;
+        const parentFetchKeys = async (batchSize, startKey)=>{
+            if (!_class_private_field_get(this, _deletedPrefix).some((dp)=>startKey.startsWith(dp))) {
+                const newKeys = [];
+                while(newKeys.length < batchSize){
+                    const remote = await _class_private_field_get(this, _parent)?.getKeysPaged(prefix, batchSize, startKey) ?? [];
+                    if (remote.length) {
+                        startKey = remote[remote.length - 1];
+                    }
+                    for (const key of remote){
+                        if (_class_private_field_get(this, _store).get(key) === "Deleted") {
+                            continue;
+                        }
+                        if (_class_private_field_get(this, _deletedPrefix).some((dp)=>key.startsWith(dp))) {
+                            continue;
+                        }
+                        newKeys.push(key);
+                    }
+                    if (remote.length < batchSize) {
+                        parentFetchComplete = true;
+                        break;
+                    }
+                }
+                return newKeys;
+            } else {
+                parentFetchComplete = true;
+                return [];
+            }
+        };
+        const res = [];
+        const foundNextKey = (key)=>{
+            // make sure keys are unique
+            if (!res.includes(key)) {
+                res.push(key);
+            }
+        };
+        const iterLocalKeys = (prefix, startKey, includeFirst, endKey)=>{
+            let idx = _class_private_field_get(this, _keys).findIndex((x)=>x.startsWith(startKey));
+            if (_class_private_field_get(this, _keys)[idx] !== startKey) {
+                idx = _class_private_field_get(this, _keys).findIndex((x)=>x.startsWith(prefix) && x > startKey);
+                const key = _class_private_field_get(this, _keys)[idx];
+                if (key) {
+                    if (endKey && key >= endKey) {
+                        return startKey;
+                    }
+                    foundNextKey(key);
+                    ++idx;
                 }
-                if (_class_private_field_get(this, _deletedPrefix).some((dp)=>key.startsWith(dp))) {
-                    continue;
+            }
+            if (idx !== -1) {
+                if (includeFirst) {
+                    const key = _class_private_field_get(this, _keys)[idx];
+                    if (key) {
+                        foundNextKey(key);
+                    }
                 }
-                _class_private_method_get(this, _addKey, addKey).call(this, key);
+                while(res.length < pageSize){
+                    ++idx;
+                    const key = _class_private_field_get(this, _keys)[idx];
+                    if (!key || !key.startsWith(prefix)) {
+                        break;
+                    }
+                    if (endKey && key >= endKey) {
+                        break;
+                    }
+                    foundNextKey(key);
+                }
+                return _lodash.default.last(res) ?? startKey;
             }
+            return startKey;
+        };
+        if (prefix !== startKey && _class_private_field_get(this, _keys).find((x)=>x === startKey)) {
+            startKey = iterLocalKeys(prefix, startKey, false);
         }
-        let idx = _lodash.default.sortedIndex(_class_private_field_get(this, _keys), startKey);
-        if (_class_private_field_get(this, _keys)[idx] === startKey) {
-            ++idx;
-        }
-        const res = [];
-        while(res.length < pageSize){
-            const key = _class_private_field_get(this, _keys)[idx];
-            if (!key || !key.startsWith(prefix)) {
-                break;
+        // then iterate the parent keys
+        let keys = await parentFetchKeys(pageSize - res.length, startKey);
+        if (keys.length) {
+            let idx = 0;
+            while(res.length < pageSize){
+                const key = keys[idx];
+                if (!key || !key.startsWith(prefix)) {
+                    if (parentFetchComplete) {
+                        break;
+                    } else {
+                        keys = await parentFetchKeys(pageSize - res.length, _lodash.default.last(keys));
+                        continue;
+                    }
+                }
+                const keyPosition = _lodash.default.sortedIndex(_class_private_field_get(this, _keys), key);
+                const localParentKey = _class_private_field_get(this, _keys)[keyPosition - 1];
+                if (localParentKey < key) {
+                    startKey = iterLocalKeys(prefix, startKey, false, key);
+                }
+                foundNextKey(key);
+                ++idx;
             }
-            res.push(key);
-            ++idx;
+        }
+        if (res.length < pageSize) {
+            iterLocalKeys(prefix, startKey, prefix === startKey);
         }
         return res;
     }

package/dist/cjs/logger.d.ts

@@ -1,14 +1,4 @@
 import { pino } from 'pino';
-export declare const pinoLogger: import("pino").Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
-export declare const defaultLogger: pino.Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
+export declare const pinoLogger: import("pino").Logger<never>;
+export declare const defaultLogger: pino.Logger<never>;
 export declare const truncate: (val: any) => any;

package/dist/cjs/rpc/shared.d.ts

@@ -1,10 +1,5 @@
 import { Blockchain } from '../blockchain/index.js';
-export declare const logger: import("pino").default.Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
+export declare const logger: import("pino").default.Logger<never>;
 export declare class ResponseError extends Error {
     code: number;
     constructor(code: number, message: string);

package/dist/cjs/utils/index.d.ts

@@ -17,6 +17,8 @@ export type Deferred<T> = {
     promise: Promise<T>;
 };
 export declare function defer<T>(): Deferred<T>;
+export declare const CHILD_PREFIX_LENGTH: number;
+export declare const PREFIX_LENGTH = 66;
 export declare const prefixedChildKey: (prefix: HexString, key: HexString) => string;
 export declare const isPrefixedChildKey: (key: HexString) => boolean;
 export declare const splitChildKey: (key: HexString) => never[] | [`0x${string}`, `0x${string}`];

package/dist/cjs/utils/index.js

@@ -9,6 +9,12 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    CHILD_PREFIX_LENGTH: function() {
+        return CHILD_PREFIX_LENGTH;
+    },
+    PREFIX_LENGTH: function() {
+        return PREFIX_LENGTH;
+    },
     compactHex: function() {
         return compactHex;
     },
@@ -111,15 +117,15 @@ function defer() {
 // The difference is that child storage keys are prefixed with the child storage key
 // :child_storage:default: as hex string
 const DEFAULT_CHILD_STORAGE = '0x3a6368696c645f73746f726167653a64656661756c743a';
-// length of the child storage key
-const CHILD_LENGTH = DEFAULT_CHILD_STORAGE.length + 64;
+const CHILD_PREFIX_LENGTH = DEFAULT_CHILD_STORAGE.length + 64;
+const PREFIX_LENGTH = 66;
 const prefixedChildKey = (prefix, key)=>prefix + (0, _hex.hexStripPrefix)(key);
 const isPrefixedChildKey = (key)=>key.startsWith(DEFAULT_CHILD_STORAGE);
 const splitChildKey = (key)=>{
     if (!key.startsWith(DEFAULT_CHILD_STORAGE)) return [];
-    if (key.length < CHILD_LENGTH) return [];
-    const child = key.slice(0, CHILD_LENGTH);
-    const rest = key.slice(CHILD_LENGTH);
+    if (key.length < CHILD_PREFIX_LENGTH) return [];
+    const child = key.slice(0, CHILD_PREFIX_LENGTH);
+    const rest = key.slice(CHILD_PREFIX_LENGTH);
     return [
         child,
         (0, _hex.hexAddPrefix)(rest)

package/dist/cjs/utils/key-cache.d.ts

@@ -1,6 +1,7 @@
 import { HexString } from '@polkadot/util/types';
-export declare const PREFIX_LENGTH = 66;
 export default class KeyCache {
+    readonly prefixLength: number;
+    constructor(prefixLength: number);
     readonly ranges: Array<{
         prefix: string;
         keys: string[];

package/dist/cjs/utils/key-cache.js

@@ -2,17 +2,9 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-function _export(target, all) {
-    for(var name in all)Object.defineProperty(target, name, {
-        enumerable: true,
-        get: all[name]
-    });
-}
-_export(exports, {
-    PREFIX_LENGTH: function() {
-        return PREFIX_LENGTH;
-    },
-    default: function() {
+Object.defineProperty(exports, "default", {
+    enumerable: true,
+    get: function() {
         return KeyCache;
     }
 });
@@ -35,21 +27,20 @@ function _interop_require_default(obj) {
         default: obj
     };
 }
-const PREFIX_LENGTH = 66;
 class KeyCache {
     feed(keys) {
-        const _keys = keys.filter((key)=>key.length >= PREFIX_LENGTH);
+        const _keys = keys.filter((key)=>key.length >= this.prefixLength);
         if (_keys.length === 0) return;
-        const startKey = _keys[0].slice(PREFIX_LENGTH);
-        const endKey = _keys[_keys.length - 1].slice(PREFIX_LENGTH);
-        const grouped = _lodash.default.groupBy(_keys, (key)=>key.slice(0, PREFIX_LENGTH));
+        const startKey = _keys[0].slice(this.prefixLength);
+        const endKey = _keys[_keys.length - 1].slice(this.prefixLength);
+        const grouped = _lodash.default.groupBy(_keys, (key)=>key.slice(0, this.prefixLength));
         for (const [prefix, keys] of Object.entries(grouped)){
             const ranges = this.ranges.filter((range)=>range.prefix === prefix);
             if (ranges.length === 0) {
                 // no existing range with prefix
                 this.ranges.push({
                     prefix,
-                    keys: keys.map((i)=>i.slice(PREFIX_LENGTH))
+                    keys: keys.map((i)=>i.slice(this.prefixLength))
                 });
                 continue;
             }
@@ -58,14 +49,14 @@ class KeyCache {
                 const startPosition = _lodash.default.sortedIndex(range.keys, startKey);
                 if (startPosition >= 0 && range.keys[startPosition] === startKey) {
                     // found existing range with prefix
-                    range.keys.splice(startPosition, keys.length, ...keys.map((i)=>i.slice(PREFIX_LENGTH)));
+                    range.keys.splice(startPosition, keys.length, ...keys.map((i)=>i.slice(this.prefixLength)));
                     merged = true;
                     break;
                 }
                 const endPosition = _lodash.default.sortedIndex(range.keys, endKey);
                 if (endPosition >= 0 && range.keys[endPosition] === endKey) {
                     // found existing range with prefix
-                    range.keys.splice(0, endPosition + 1, ...keys.map((i)=>i.slice(PREFIX_LENGTH)));
+                    range.keys.splice(0, endPosition + 1, ...keys.map((i)=>i.slice(this.prefixLength)));
                     merged = true;
                     break;
                 }
@@ -74,17 +65,25 @@ class KeyCache {
             if (!merged) {
                 this.ranges.push({
                     prefix,
-                    keys: keys.map((i)=>i.slice(PREFIX_LENGTH))
+                    keys: keys.map((i)=>i.slice(this.prefixLength))
                 });
             }
         }
     // TODO: merge ranges if they overlap
     }
     async next(startKey) {
-        if (startKey.length < PREFIX_LENGTH) return;
-        const prefix = startKey.slice(0, PREFIX_LENGTH);
-        const key = startKey.slice(PREFIX_LENGTH);
+        if (startKey.length < this.prefixLength) return;
+        const prefix = startKey.slice(0, this.prefixLength);
+        const key = startKey.slice(this.prefixLength);
         for (const range of this.ranges.filter((range)=>range.prefix === prefix)){
+            if (key.length === 0) {
+                // if key is empty then find the range with first key empty
+                if (range.keys[0] !== '') continue;
+                return [
+                    prefix,
+                    range.keys[1]
+                ].join('');
+            }
             const index = _lodash.default.sortedIndex(range.keys, key);
             if (range.keys[index] !== key) continue;
             const nextKey = range.keys[index + 1];
@@ -96,7 +95,10 @@ class KeyCache {
             }
         }
     }
-    constructor(){
-        _define_property(this, "ranges", []);
+    constructor(prefixLength){
+        _define_property(this, "prefixLength", void 0);
+        _define_property(this, "ranges", void 0);
+        this.prefixLength = prefixLength;
+        this.ranges = [];
     }
 }

package/dist/cjs/wasm-executor/index.js

@@ -44,9 +44,8 @@ const _comlink = /*#__PURE__*/ _interop_require_wildcard(require("comlink"));
 const _util = require("@polkadot/util");
 const _utilcrypto = require("@polkadot/util-crypto");
 const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
-const _keycache = require("../utils/key-cache.js");
-const _logger = require("../logger.js");
 const _index = require("../utils/index.js");
+const _logger = require("../logger.js");
 function _interop_require_default(obj) {
     return obj && obj.__esModule ? obj : {
         default: obj
@@ -160,7 +159,7 @@ const taskHandler = (block)=>{
         },
         getNextKey: async function(prefix, key) {
             const [nextKey] = await block.getKeysPaged({
-                prefix: prefix.length === 2 /** 0x */  ? key.slice(0, _keycache.PREFIX_LENGTH) : prefix,
+                prefix: prefix.length === 2 /** 0x */  ? key.slice(0, _index.PREFIX_LENGTH) : prefix,
                 pageSize: 1,
                 startKey: key
             });

package/dist/cjs/xcm/index.d.ts

@@ -1,9 +1,4 @@
 import { Blockchain } from '../blockchain/index.js';
-export declare const xcmLogger: import("pino").default.Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
+export declare const xcmLogger: import("pino").default.Logger<never>;
 export declare const connectVertical: (relaychain: Blockchain, parachain: Blockchain) => Promise<void>;
 export declare const connectParachains: (parachains: Blockchain[]) => Promise<void>;

package/dist/esm/api.d.ts

@@ -50,6 +50,7 @@ export declare class Api {
     getBlock(hash?: string): Promise<SignedBlock | null>;
     getStorage(key: string, hash?: string): Promise<`0x${string}` | null>;
     getKeysPaged(prefix: string, pageSize: number, startKey: string, hash?: string): Promise<string[]>;
+    getStorageBatch(prefix: HexString, keys: HexString[], hash?: HexString): Promise<[`0x${string}`, `0x${string}` | null][]>;
     subscribeRemoteNewHeads(cb: ProviderInterfaceCallback): Promise<string | number>;
     subscribeRemoteFinalizedHeads(cb: ProviderInterfaceCallback): Promise<string | number>;
 }

package/dist/esm/api.js

@@ -1,4 +1,5 @@
 import { prefixedChildKey, splitChildKey, stripChildPrefix } from './utils/index.js';
+import _ from 'lodash';
 /**
  * API class. Calls provider to get on-chain data.
  * Either `endpoint` or `genesis` porvider must be provided.
@@ -119,6 +120,26 @@ import { prefixedChildKey, splitChildKey, stripChildPrefix } from './utils/index
             return this.#provider.send('state_getKeysPaged', params, !!hash);
         }
     }
+    async getStorageBatch(prefix, keys, hash) {
+        const [child] = splitChildKey(prefix);
+        if (child) {
+            // child storage key, use childstate_getStorageEntries
+            // strip child prefix from keys
+            const params = [
+                child,
+                keys.map((key)=>stripChildPrefix(key))
+            ];
+            if (hash) params.push(hash);
+            return this.#provider.send('childstate_getStorageEntries', params, !!hash).then((values)=>_.zip(keys, values));
+        } else {
+            // main storage key, use state_getStorageAt
+            const params = [
+                keys
+            ];
+            if (hash) params.push(hash);
+            return this.#provider.send('state_queryStorageAt', params, !!hash).then((result)=>result[0]?.['changes'] || []);
+        }
+    }
     async subscribeRemoteNewHeads(cb) {
         if (!this.#provider.hasSubscriptions) {
             throw new Error('subscribeRemoteNewHeads only works with subscriptions');

package/dist/esm/blockchain/block.js

@@ -129,12 +129,10 @@ import { getRuntimeVersion, runTask, taskHandler } from '../wasm-executor/index.
     /**
    * Get paged storage keys.
    */ async getKeysPaged(options) {
-        const layer = new StorageLayer(this.storage);
-        await layer.fold();
         const prefix = options.prefix ?? '0x';
         const startKey = options.startKey ?? '0x';
         const pageSize = options.pageSize;
-        return layer.getKeysPaged(prefix, pageSize, startKey);
+        return this.storage.getKeysPaged(prefix, pageSize, startKey);
     }
     /**
    * Push a layer to the storage stack.

package/dist/esm/blockchain/storage-layer.js

@@ -1,6 +1,7 @@
 import _ from 'lodash';
+import { CHILD_PREFIX_LENGTH, PREFIX_LENGTH, isPrefixedChildKey } from '../utils/index.js';
 import { defaultLogger } from '../logger.js';
-import KeyCache, { PREFIX_LENGTH } from '../utils/key-cache.js';
+import KeyCache from '../utils/key-cache.js';
 const logger = defaultLogger.child({
     name: 'layer'
 });
@@ -14,7 +15,8 @@ export class RemoteStorageLayer {
     #api;
     #at;
     #db;
-    #keyCache = new KeyCache();
+    #keyCache = new KeyCache(PREFIX_LENGTH);
+    #defaultChildKeyCache = new KeyCache(CHILD_PREFIX_LENGTH);
     constructor(api, at, db){
         this.#api = api;
         this.#at = at;
@@ -47,14 +49,16 @@ export class RemoteStorageLayer {
             pageSize,
             startKey
         }, 'RemoteStorageLayer getKeysPaged');
+        const isChild = isPrefixedChildKey(prefix);
+        const minPrefixLen = isChild ? CHILD_PREFIX_LENGTH : PREFIX_LENGTH;
         // can't handle keyCache without prefix
-        if (prefix.length < PREFIX_LENGTH || startKey.length < PREFIX_LENGTH) {
+        if (prefix.length < minPrefixLen || startKey.length < minPrefixLen) {
             return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at);
         }
         let batchComplete = false;
         const keysPaged = [];
         while(keysPaged.length < pageSize){
-            const nextKey = await this.#keyCache.next(startKey);
+            const nextKey = isChild ? await this.#defaultChildKeyCache.next(startKey) : await this.#keyCache.next(startKey);
             if (nextKey) {
                 keysPaged.push(nextKey);
                 startKey = nextKey;
@@ -69,14 +73,29 @@ export class RemoteStorageLayer {
             batchComplete = batch.length < BATCH_SIZE;
             // feed the key cache
             if (batch.length > 0) {
-                this.#keyCache.feed([
-                    startKey,
-                    ...batch
-                ]);
+                if (isChild) {
+                    this.#defaultChildKeyCache.feed([
+                        startKey,
+                        ...batch
+                    ]);
+                } else {
+                    this.#keyCache.feed([
+                        startKey,
+                        ...batch
+                    ]);
+                }
             }
             if (batch.length === 0) {
                 break;
             }
+            if (this.#db) {
+                // batch fetch storage values and save to db, they may be used later
+                this.#api.getStorageBatch(prefix, batch, this.#at).then((storage)=>{
+                    for (const [key, value] of storage){
+                        this.#db.saveStorage(this.#at, key, value);
+                    }
+                });
+            }
         }
         return keysPaged;
     }
@@ -169,30 +188,105 @@ export class StorageLayer {
         }
     }
     async getKeysPaged(prefix, pageSize, startKey) {
-        if (!this.#deletedPrefix.some((dp)=>startKey.startsWith(dp))) {
-            const remote = await this.#parent?.getKeysPaged(prefix, pageSize, startKey) ?? [];
-            for (const key of remote){
-                if (this.#store.get(key) === "Deleted") {
-                    continue;
+        let parentFetchComplete = false;
+        const parentFetchKeys = async (batchSize, startKey)=>{
+            if (!this.#deletedPrefix.some((dp)=>startKey.startsWith(dp))) {
+                const newKeys = [];
+                while(newKeys.length < batchSize){
+                    const remote = await this.#parent?.getKeysPaged(prefix, batchSize, startKey) ?? [];
+                    if (remote.length) {
+                        startKey = remote[remote.length - 1];
+                    }
+                    for (const key of remote){
+                        if (this.#store.get(key) === "Deleted") {
+                            continue;
+                        }
+                        if (this.#deletedPrefix.some((dp)=>key.startsWith(dp))) {
+                            continue;
+                        }
+                        newKeys.push(key);
+                    }
+                    if (remote.length < batchSize) {
+                        parentFetchComplete = true;
+                        break;
+                    }
                 }
-                if (this.#deletedPrefix.some((dp)=>key.startsWith(dp))) {
-                    continue;
+                return newKeys;
+            } else {
+                parentFetchComplete = true;
+                return [];
+            }
+        };
+        const res = [];
+        const foundNextKey = (key)=>{
+            // make sure keys are unique
+            if (!res.includes(key)) {
+                res.push(key);
+            }
+        };
+        const iterLocalKeys = (prefix, startKey, includeFirst, endKey)=>{
+            let idx = this.#keys.findIndex((x)=>x.startsWith(startKey));
+            if (this.#keys[idx] !== startKey) {
+                idx = this.#keys.findIndex((x)=>x.startsWith(prefix) && x > startKey);
+                const key = this.#keys[idx];
+                if (key) {
+                    if (endKey && key >= endKey) {
+                        return startKey;
+                    }
+                    foundNextKey(key);
+                    ++idx;
                 }
-                this.#addKey(key);
             }
+            if (idx !== -1) {
+                if (includeFirst) {
+                    const key = this.#keys[idx];
+                    if (key) {
+                        foundNextKey(key);
+                    }
+                }
+                while(res.length < pageSize){
+                    ++idx;
+                    const key = this.#keys[idx];
+                    if (!key || !key.startsWith(prefix)) {
+                        break;
+                    }
+                    if (endKey && key >= endKey) {
+                        break;
+                    }
+                    foundNextKey(key);
+                }
+                return _.last(res) ?? startKey;
+            }
+            return startKey;
+        };
+        if (prefix !== startKey && this.#keys.find((x)=>x === startKey)) {
+            startKey = iterLocalKeys(prefix, startKey, false);
         }
-        let idx = _.sortedIndex(this.#keys, startKey);
-        if (this.#keys[idx] === startKey) {
-            ++idx;
-        }
-        const res = [];
-        while(res.length < pageSize){
-            const key = this.#keys[idx];
-            if (!key || !key.startsWith(prefix)) {
-                break;
+        // then iterate the parent keys
+        let keys = await parentFetchKeys(pageSize - res.length, startKey);
+        if (keys.length) {
+            let idx = 0;
+            while(res.length < pageSize){
+                const key = keys[idx];
+                if (!key || !key.startsWith(prefix)) {
+                    if (parentFetchComplete) {
+                        break;
+                    } else {
+                        keys = await parentFetchKeys(pageSize - res.length, _.last(keys));
+                        continue;
+                    }
+                }
+                const keyPosition = _.sortedIndex(this.#keys, key);
+                const localParentKey = this.#keys[keyPosition - 1];
+                if (localParentKey < key) {
+                    startKey = iterLocalKeys(prefix, startKey, false, key);
+                }
+                foundNextKey(key);
+                ++idx;
             }
-            res.push(key);
-            ++idx;
+        }
+        if (res.length < pageSize) {
+            iterLocalKeys(prefix, startKey, prefix === startKey);
         }
         return res;
     }

package/dist/esm/logger.d.ts

@@ -1,14 +1,4 @@
 import { pino } from 'pino';
-export declare const pinoLogger: import("pino").Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
-export declare const defaultLogger: pino.Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
+export declare const pinoLogger: import("pino").Logger<never>;
+export declare const defaultLogger: pino.Logger<never>;
 export declare const truncate: (val: any) => any;

package/dist/esm/rpc/shared.d.ts

@@ -1,10 +1,5 @@
 import { Blockchain } from '../blockchain/index.js';
-export declare const logger: import("pino").default.Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
+export declare const logger: import("pino").default.Logger<never>;
 export declare class ResponseError extends Error {
     code: number;
     constructor(code: number, message: string);

package/dist/esm/utils/index.d.ts

@@ -17,6 +17,8 @@ export type Deferred<T> = {
     promise: Promise<T>;
 };
 export declare function defer<T>(): Deferred<T>;
+export declare const CHILD_PREFIX_LENGTH: number;
+export declare const PREFIX_LENGTH = 66;
 export declare const prefixedChildKey: (prefix: HexString, key: HexString) => string;
 export declare const isPrefixedChildKey: (key: HexString) => boolean;
 export declare const splitChildKey: (key: HexString) => never[] | [`0x${string}`, `0x${string}`];

package/dist/esm/utils/index.js

@@ -57,7 +57,9 @@ export function defer() {
 // :child_storage:default: as hex string
 const DEFAULT_CHILD_STORAGE = '0x3a6368696c645f73746f726167653a64656661756c743a';
 // length of the child storage key
-const CHILD_LENGTH = DEFAULT_CHILD_STORAGE.length + 64;
+export const CHILD_PREFIX_LENGTH = DEFAULT_CHILD_STORAGE.length + 64;
+// 0x + 32 module + 32 method
+export const PREFIX_LENGTH = 66;
 // returns a key that is prefixed with the child storage key
 export const prefixedChildKey = (prefix, key)=>prefix + hexStripPrefix(key);
 // returns true if the key is a child storage key
@@ -65,9 +67,9 @@ export const isPrefixedChildKey = (key)=>key.startsWith(DEFAULT_CHILD_STORAGE);
 // returns a key that is split into the child storage key and the rest
 export const splitChildKey = (key)=>{
     if (!key.startsWith(DEFAULT_CHILD_STORAGE)) return [];
-    if (key.length < CHILD_LENGTH) return [];
-    const child = key.slice(0, CHILD_LENGTH);
-    const rest = key.slice(CHILD_LENGTH);
+    if (key.length < CHILD_PREFIX_LENGTH) return [];
+    const child = key.slice(0, CHILD_PREFIX_LENGTH);
+    const rest = key.slice(CHILD_PREFIX_LENGTH);
     return [
         child,
         hexAddPrefix(rest)

package/dist/esm/utils/key-cache.d.ts

@@ -1,6 +1,7 @@
 import { HexString } from '@polkadot/util/types';
-export declare const PREFIX_LENGTH = 66;
 export default class KeyCache {
+    readonly prefixLength: number;
+    constructor(prefixLength: number);
     readonly ranges: Array<{
         prefix: string;
         keys: string[];

package/dist/esm/utils/key-cache.js

@@ -1,21 +1,24 @@
 import _ from 'lodash';
-// 0x + 32 module + 32 method
-export const PREFIX_LENGTH = 66;
 export default class KeyCache {
-    ranges = [];
+    prefixLength;
+    constructor(prefixLength){
+        this.prefixLength = prefixLength;
+        this.ranges = [];
+    }
+    ranges;
     feed(keys) {
-        const _keys = keys.filter((key)=>key.length >= PREFIX_LENGTH);
+        const _keys = keys.filter((key)=>key.length >= this.prefixLength);
         if (_keys.length === 0) return;
-        const startKey = _keys[0].slice(PREFIX_LENGTH);
-        const endKey = _keys[_keys.length - 1].slice(PREFIX_LENGTH);
-        const grouped = _.groupBy(_keys, (key)=>key.slice(0, PREFIX_LENGTH));
+        const startKey = _keys[0].slice(this.prefixLength);
+        const endKey = _keys[_keys.length - 1].slice(this.prefixLength);
+        const grouped = _.groupBy(_keys, (key)=>key.slice(0, this.prefixLength));
         for (const [prefix, keys] of Object.entries(grouped)){
             const ranges = this.ranges.filter((range)=>range.prefix === prefix);
             if (ranges.length === 0) {
                 // no existing range with prefix
                 this.ranges.push({
                     prefix,
-                    keys: keys.map((i)=>i.slice(PREFIX_LENGTH))
+                    keys: keys.map((i)=>i.slice(this.prefixLength))
                 });
                 continue;
             }
@@ -24,14 +27,14 @@ export default class KeyCache {
                 const startPosition = _.sortedIndex(range.keys, startKey);
                 if (startPosition >= 0 && range.keys[startPosition] === startKey) {
                     // found existing range with prefix
-                    range.keys.splice(startPosition, keys.length, ...keys.map((i)=>i.slice(PREFIX_LENGTH)));
+                    range.keys.splice(startPosition, keys.length, ...keys.map((i)=>i.slice(this.prefixLength)));
                     merged = true;
                     break;
                 }
                 const endPosition = _.sortedIndex(range.keys, endKey);
                 if (endPosition >= 0 && range.keys[endPosition] === endKey) {
                     // found existing range with prefix
-                    range.keys.splice(0, endPosition + 1, ...keys.map((i)=>i.slice(PREFIX_LENGTH)));
+                    range.keys.splice(0, endPosition + 1, ...keys.map((i)=>i.slice(this.prefixLength)));
                     merged = true;
                     break;
                 }
@@ -40,17 +43,25 @@ export default class KeyCache {
             if (!merged) {
                 this.ranges.push({
                     prefix,
-                    keys: keys.map((i)=>i.slice(PREFIX_LENGTH))
+                    keys: keys.map((i)=>i.slice(this.prefixLength))
                 });
             }
         }
     // TODO: merge ranges if they overlap
     }
     async next(startKey) {
-        if (startKey.length < PREFIX_LENGTH) return;
-        const prefix = startKey.slice(0, PREFIX_LENGTH);
-        const key = startKey.slice(PREFIX_LENGTH);
+        if (startKey.length < this.prefixLength) return;
+        const prefix = startKey.slice(0, this.prefixLength);
+        const key = startKey.slice(this.prefixLength);
         for (const range of this.ranges.filter((range)=>range.prefix === prefix)){
+            if (key.length === 0) {
+                // if key is empty then find the range with first key empty
+                if (range.keys[0] !== '') continue;
+                return [
+                    prefix,
+                    range.keys[1]
+                ].join('');
+            }
             const index = _.sortedIndex(range.keys, key);
             if (range.keys[index] !== key) continue;
             const nextKey = range.keys[index + 1];

package/dist/esm/wasm-executor/index.js

@@ -2,9 +2,8 @@ import * as Comlink from 'comlink';
 import { hexToString, hexToU8a, u8aToBn } from '@polkadot/util';
 import { randomAsHex } from '@polkadot/util-crypto';
 import _ from 'lodash';
-import { PREFIX_LENGTH } from '../utils/key-cache.js';
+import { PREFIX_LENGTH, stripChildPrefix } from '../utils/index.js';
 import { defaultLogger, truncate } from '../logger.js';
-import { stripChildPrefix } from '../utils/index.js';
 const logger = defaultLogger.child({
     name: 'executor'
 });

package/dist/esm/xcm/index.d.ts

@@ -1,9 +1,4 @@
 import { Blockchain } from '../blockchain/index.js';
-export declare const xcmLogger: import("pino").default.Logger<{
-    level: string;
-    transport: {
-        target: string;
-    };
-}>;
+export declare const xcmLogger: import("pino").default.Logger<never>;
 export declare const connectVertical: (relaychain: Blockchain, parachain: Blockchain) => Promise<void>;
 export declare const connectParachains: (parachains: Blockchain[]) => Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@acala-network/chopsticks-core",
-  "version": "0.9.6",
+  "version": "0.9.8-1",
   "author": "Acala Developers <hello@acala.network>",
   "license": "Apache-2.0",
   "type": "module",
@@ -12,28 +12,28 @@
     "docs:prep": "typedoc"
   },
   "dependencies": {
-    "@acala-network/chopsticks-executor": "0.9.6",
-    "@polkadot/rpc-provider": "^10.10.1",
-    "@polkadot/types": "^10.10.1",
-    "@polkadot/types-codec": "^10.10.1",
-    "@polkadot/types-known": "^10.10.1",
-    "@polkadot/util": "^12.5.1",
-    "@polkadot/util-crypto": "^12.5.1",
+    "@acala-network/chopsticks-executor": "0.9.8-1",
+    "@polkadot/rpc-provider": "^10.11.2",
+    "@polkadot/types": "^10.11.2",
+    "@polkadot/types-codec": "^10.11.2",
+    "@polkadot/types-known": "^10.11.2",
+    "@polkadot/util": "^12.6.2",
+    "@polkadot/util-crypto": "^12.6.2",
     "comlink": "^4.4.1",
     "eventemitter3": "^5.0.1",
     "lodash": "^4.17.21",
     "lru-cache": "^10.1.0",
-    "pino": "^8.16.2",
-    "pino-pretty": "^10.2.3",
+    "pino": "^8.17.2",
+    "pino-pretty": "^10.3.1",
     "rxjs": "^7.8.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@swc/cli": "0.1.63",
-    "@swc/core": "^1.3.100",
+    "@swc/cli": "0.1.65",
+    "@swc/core": "^1.3.105",
     "@types/lodash": "^4.14.202",
     "typescript": "^5.3.3",
-    "vitest": "^1.0.4"
+    "vitest": "^1.2.1"
   },
   "files": [
     "dist/esm/**",