@acala-network/chopsticks 0.6.4 → 0.6.5

package/lib/blockchain/storage-layer.js

@@ -6,11 +6,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.StorageLayer = exports.RemoteStorageLayer = void 0;
 const lodash_1 = __importDefault(require("lodash"));
 const logger_1 = require("../logger");
+const key_cache_1 = __importDefault(require("../utils/key-cache"));
 const logger = logger_1.defaultLogger.child({ name: 'layer' });
+const BATCH_SIZE = 1000;
 class RemoteStorageLayer {
     #api;
     #at;
     #db;
+    #keyCache = new key_cache_1.default();
     constructor(api, at, db) {
         this.#api = api;
         this.#at = at;
@@ -33,8 +36,39 @@ class RemoteStorageLayer {
     }
     async fold() { }
     async getKeysPaged(prefix, pageSize, startKey) {
+        if (pageSize > BATCH_SIZE)
+            throw new Error(`pageSize must be less or equal to ${BATCH_SIZE}`);
         logger.trace({ at: this.#at, prefix, pageSize, startKey }, 'RemoteStorageLayer getKeysPaged');
-        return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at);
+        // can't handle keyCache without prefix
+        if (prefix.length < 66) {
+            return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at);
+        }
+        let batchComplete = false;
+        const keysPaged = [];
+        while (keysPaged.length < pageSize) {
+            const nextKey = await this.#keyCache.next(startKey);
+            if (nextKey) {
+                keysPaged.push(nextKey);
+                startKey = nextKey;
+                continue;
+            }
+            // batch fetch was completed
+            if (batchComplete) {
+                break;
+            }
+            // fetch a batch of keys
+            const batch = await this.#api.getKeysPaged(prefix, BATCH_SIZE, startKey, this.#at);
+            batchComplete = batch.length < BATCH_SIZE;
+            // feed the key cache
+            if (batch.length > 0) {
+                this.#keyCache.feed([startKey, ...batch]);
+            }
+            if (batch.length === 0) {
+                // no more keys were found
+                break;
+            }
+        }
+        return keysPaged;
     }
 }
 exports.RemoteStorageLayer = RemoteStorageLayer;
@@ -128,7 +162,6 @@ class StorageLayer {
     }
     async getKeysPaged(prefix, pageSize, startKey) {
         if (!this.#deletedPrefix.some((prefix) => startKey.startsWith(prefix))) {
-            // TODO: maintain a list of fetched ranges to avoid fetching the same range multiple times
             const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? [];
             for (const key of remote) {
                 if (this.#deletedPrefix.some((prefix) => key.startsWith(prefix))) {

package/lib/executor.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getAuraSlotDuration = exports.emptyTaskHandler = exports.taskHandler = exports.runTask = exports.createProof = exports.decodeProof = exports.calculateStateRoot = exports.getRuntimeVersion = void 0;
 const util_1 = require("@polkadot/util");
+const key_cache_1 = require("./utils/key-cache");
 const chopsticks_executor_1 = require("@acala-network/chopsticks-executor");
 const logger_1 = require("./logger");
 const lodash_1 = __importDefault(require("lodash"));
@@ -48,16 +49,24 @@ const runTask = async (task, callback = exports.emptyTaskHandler) => {
 };
 exports.runTask = runTask;
 const taskHandler = (block) => {
+    const batchSize = 1000;
     return {
         getStorage: async function (key) {
             return block.get(key);
         },
         getPrefixKeys: async function (key) {
-            return block.getKeysPaged({ prefix: key, pageSize: 1000, startKey: key });
+            let keys = [];
+            let startKey = key;
+            while (startKey) {
+                const batch = await block.getKeysPaged({ prefix: key.slice(0, key_cache_1.PREFIX_LENGTH), pageSize: batchSize, startKey });
+                keys = keys.concat(batch);
+                startKey = batch[batchSize - 1];
+            }
+            return keys;
         },
         getNextKey: async function (key) {
-            const keys = await block.getKeysPaged({ prefix: '0x', pageSize: 1, startKey: key });
-            return keys[0];
+            const [nextKey] = await block.getKeysPaged({ prefix: key.slice(0, key_cache_1.PREFIX_LENGTH), pageSize: 1, startKey: key });
+            return nextKey;
         },
     };
 };

package/lib/rpc/shared.d.ts

@@ -4,7 +4,7 @@ export declare const logger: import("pino").default.Logger<{
     transport: {
         target: string;
     };
-} & import("pino").default.ChildLoggerOptions>;
+}>;
 export declare class ResponseError extends Error {
     code: number;
     constructor(code: number, message: string);

package/lib/utils/key-cache.d.ts

@@ -0,0 +1,10 @@
+import { HexString } from '@polkadot/util/types';
+export declare const PREFIX_LENGTH = 66;
+export default class KeyCache {
+    readonly ranges: Array<{
+        prefix: string;
+        keys: string[];
+    }>;
+    feed(keys: HexString[]): void;
+    next(startKey: HexString): Promise<HexString | undefined>;
+}

package/lib/utils/key-cache.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PREFIX_LENGTH = void 0;
+const lodash_1 = __importDefault(require("lodash"));
+// 0x + 32 module + 32 method
+exports.PREFIX_LENGTH = 66;
+class KeyCache {
+    ranges = [];
+    feed(keys) {
+        const _keys = keys.filter((key) => key.length >= exports.PREFIX_LENGTH);
+        if (_keys.length === 0)
+            return;
+        const startKey = _keys[0].slice(exports.PREFIX_LENGTH);
+        const endKey = _keys[_keys.length - 1].slice(exports.PREFIX_LENGTH);
+        const grouped = lodash_1.default.groupBy(_keys, (key) => key.slice(0, exports.PREFIX_LENGTH));
+        for (const [prefix, keys] of Object.entries(grouped)) {
+            const ranges = this.ranges.filter((range) => range.prefix === prefix);
+            if (ranges.length === 0) {
+                // no existing range with prefix
+                this.ranges.push({ prefix, keys: keys.map((i) => i.slice(exports.PREFIX_LENGTH)) });
+                continue;
+            }
+            let merged = false;
+            for (const range of ranges) {
+                const startPosition = lodash_1.default.sortedIndex(range.keys, startKey);
+                if (startPosition >= 0 && range.keys[startPosition] === startKey) {
+                    // found existing range with prefix
+                    range.keys.splice(startPosition, keys.length, ...keys.map((i) => i.slice(exports.PREFIX_LENGTH)));
+                    merged = true;
+                    break;
+                }
+                const endPosition = lodash_1.default.sortedIndex(range.keys, endKey);
+                if (endPosition >= 0 && range.keys[endPosition] === endKey) {
+                    // found existing range with prefix
+                    range.keys.splice(0, endPosition + 1, ...keys.map((i) => i.slice(exports.PREFIX_LENGTH)));
+                    merged = true;
+                    break;
+                }
+            }
+            // insert new prefix with range
+            if (!merged) {
+                this.ranges.push({ prefix, keys: keys.map((i) => i.slice(exports.PREFIX_LENGTH)) });
+            }
+        }
+        // TODO: merge ranges if they overlap
+    }
+    async next(startKey) {
+        if (startKey.length < exports.PREFIX_LENGTH)
+            return;
+        const prefix = startKey.slice(0, exports.PREFIX_LENGTH);
+        const key = startKey.slice(exports.PREFIX_LENGTH);
+        for (const range of this.ranges.filter((range) => range.prefix === prefix)) {
+            const index = lodash_1.default.sortedIndex(range.keys, key);
+            if (range.keys[index] !== key)
+                continue;
+            const nextKey = range.keys[index + 1];
+            if (nextKey) {
+                return [prefix, nextKey].join('');
+            }
+        }
+    }
+}
+exports.default = KeyCache;

package/lib/xcm/index.d.ts

@@ -4,6 +4,6 @@ export declare const logger: import("pino").default.Logger<{
     transport: {
         target: string;
     };
-} & import("pino").default.ChildLoggerOptions>;
+}>;
 export declare const connectVertical: (relaychain: Blockchain, parachain: Blockchain) => Promise<void>;
 export declare const connectParachains: (parachains: Blockchain[]) => Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@acala-network/chopsticks",
-  "version": "0.6.4",
+  "version": "0.6.5",
   "author": "Bryan Chen <xlchen1291@gmail.com>",
   "license": "Apache-2.0",
   "bin": "./chopsticks.js",
@@ -16,17 +16,17 @@
     "dev:moonbeam": "cd ../..; ts-node-dev --transpile-only --inspect --notify=false packages/chopsticks/src/cli.ts -- --config=configs/moonbeam.yml"
   },
   "dependencies": {
-    "@acala-network/chopsticks-executor": "0.6.4",
-    "@polkadot/api": "^10.5.1",
+    "@acala-network/chopsticks-executor": "0.6.5",
+    "@polkadot/api": "^10.7.2",
     "axios": "^1.4.0",
     "js-yaml": "^4.1.0",
     "jsondiffpatch": "^0.4.1",
     "lodash": "^4.17.21",
-    "pino": "^8.11.0",
+    "pino": "^8.14.1",
     "pino-pretty": "^10.0.0",
     "reflect-metadata": "^0.1.13",
     "sqlite3": "^5.1.6",
-    "typeorm": "^0.3.15",
+    "typeorm": "^0.3.16",
     "ws": "^8.13.0",
     "yargs": "^17.7.2",
     "zod": "^3.21.4"
@@ -34,7 +34,7 @@
   "devDependencies": {
     "@types/js-yaml": "^4.0.5",
     "@types/lodash": "^4.14.194",
-    "@types/node": "^18.16.3",
+    "@types/node": "^20.2.3",
     "@types/ws": "^8.5.4",
     "@types/yargs": "^17.0.24",
     "ts-node": "^10.9.1",