@acala-network/chopsticks 0.10.2 → 0.12.0

package/dist/cjs/utils/fetch-storages.js

@@ -0,0 +1,213 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: all[name]
+    });
+}
+_export(exports, {
+    fetchStorages: function() {
+        return fetchStorages;
+    },
+    getPrefixesFromConfig: function() {
+        return getPrefixesFromConfig;
+    },
+    logger: function() {
+        return logger;
+    },
+    startFetchStorageWorker: function() {
+        return startFetchStorageWorker;
+    }
+});
+const _chopstickscore = require("@acala-network/chopsticks-core");
+const _api = require("@polkadot/api");
+const _chopsticksdb = require("@acala-network/chopsticks-db");
+const _rpcprovider = require("@polkadot/rpc-provider");
+const _util = require("@polkadot/util");
+const _types = require("@polkadot/types");
+const _comlink = require("comlink");
+const _utilcrypto = require("@polkadot/util-crypto");
+const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
+const _nodeadapter = /*#__PURE__*/ _interop_require_default(require("comlink/dist/umd/node-adapter.js"));
+const _nodeworker_threads = /*#__PURE__*/ _interop_require_default(require("node:worker_threads"));
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+const BATCH_SIZE = 1000;
+const logger = _chopstickscore.defaultLogger.child({
+    name: 'fetch-storages'
+});
+const getHexKeyWithArgs = (meta, storage, args)=>{
+    const isPartialKey = args.length !== (meta.type.isPlain ? 0 : meta.type.asMap.hashers.length);
+    const hexKey = isPartialKey && storage.creator.iterKey ? storage.creator.iterKey(...args).toHex() : (0, _util.u8aToHex)((0, _util.compactStripLength)(storage.creator(...args))[1]);
+    return hexKey;
+};
+const checkPalletStorageByName = (meta, palletName, storageName)=>{
+    const pallet = meta.query[(0, _util.stringCamelCase)(palletName)];
+    if (!pallet) throw Error(`Cannot find pallet ${palletName}`);
+    let storage;
+    if (storageName) {
+        storage = pallet[(0, _util.stringCamelCase)(storageName)];
+        if (!storage) throw Error(`Cannot find storage ${storageName} in pallet ${palletName}`);
+    }
+    return {
+        pallet,
+        storage
+    };
+};
+const getPrefixesFromConfig = async (config, api)=>{
+    logger.debug({
+        config
+    }, 'received fetch-storage config');
+    const prefixes = [];
+    const metadata = await api.rpc.state.getMetadata();
+    const expandMeta = (0, _types.expandMetadata)(metadata.registry, metadata);
+    for (const item of config){
+        if (typeof item === 'string' && item.startsWith('0x')) {
+            // hex
+            prefixes.push(item);
+        } else if (typeof item === 'string' && !item.includes('.')) {
+            // pallet
+            checkPalletStorageByName(expandMeta, item);
+            prefixes.push((0, _utilcrypto.xxhashAsHex)(item, 128));
+        } else if (typeof item === 'string' && item.includes('.')) {
+            // pallet.storage
+            const [palletName, storageName] = item.split('.');
+            const { storage } = checkPalletStorageByName(expandMeta, palletName, storageName);
+            prefixes.push((0, _util.u8aToHex)(storage.keyPrefix()));
+        } else if (typeof item === 'object') {
+            // object cases
+            const [objectKey, objectVal] = Object.entries(item)[0];
+            if (typeof objectVal === 'string') {
+                // - System: Account
+                const { storage } = checkPalletStorageByName(expandMeta, objectKey, objectVal);
+                prefixes.push((0, _util.u8aToHex)(storage.keyPrefix()));
+            } else if (objectKey.includes('.') && Array.isArray(objectVal)) {
+                // - Pallet.Storage: [xxx, ...]
+                const [pallet, storage] = objectKey.split('.').map((x)=>(0, _util.stringCamelCase)(x));
+                checkPalletStorageByName(expandMeta, pallet, storage);
+                const storageEntry = api.query[pallet][storage];
+                const meta = storageEntry.creator.meta;
+                const args = objectVal;
+                const hexKey = getHexKeyWithArgs(meta, storageEntry, args);
+                prefixes.push(hexKey);
+            } else if (!Array.isArray(objectVal)) {
+                // - Tokens:
+                //     Accounts: [xxx, ...]
+                const pallet = (0, _util.stringCamelCase)(objectKey);
+                const [storage, args] = Object.entries(objectVal)[0];
+                checkPalletStorageByName(expandMeta, pallet, storage);
+                const storageEntry = api.query[pallet][(0, _util.stringCamelCase)(storage)];
+                const meta = storageEntry.creator.meta;
+                const hexKey = getHexKeyWithArgs(meta, storageEntry, args);
+                prefixes.push(hexKey);
+            } else {
+                throw new Error(`Unsupported fetch-storage config: ${objectKey}.${objectVal}`);
+            }
+        }
+    }
+    logger.debug({
+        prefixes
+    }, 'prefixes from config');
+    return prefixes;
+};
+const fetchStorages = async ({ block, endpoint, dbPath, config })=>{
+    if (!endpoint) throw new Error('endpoint is required');
+    if (!block) throw new Error('block is required');
+    const provider = new _rpcprovider.WsProvider(endpoint, 3_000);
+    const apiPromise = new _api.ApiPromise({
+        provider
+    });
+    await apiPromise.isReady;
+    let blockHash;
+    if (block == null) {
+        const lastHdr = await apiPromise.rpc.chain.getHeader();
+        blockHash = lastHdr.hash.toString();
+    } else if (typeof block === 'string' && block.startsWith('0x')) {
+        blockHash = block;
+    } else if (Number.isInteger(+block)) {
+        blockHash = await apiPromise.rpc.chain.getBlockHash(Number(block)).then((h)=>h.toString());
+    } else {
+        throw new Error(`Invalid block number or hash: ${block}`);
+    }
+    const prefixesFromConfig = await getPrefixesFromConfig(config, apiPromise);
+    const uniqPrefixes = _lodash.default.uniq(prefixesFromConfig);
+    const processPrefixes = (prefixes)=>{
+        prefixes.sort();
+        const result = [];
+        for (const prefix of prefixes){
+            // check if the current prefix is not a prefix of any added prefix
+            if (!result.some((prev)=>prefix.startsWith(prev))) {
+                result.push(prefix);
+            }
+        }
+        return result;
+    };
+    const prefixes = processPrefixes(uniqPrefixes);
+    if (!prefixes.length) throw new Error('No prefixes to fetch');
+    const signedBlock = await apiPromise.rpc.chain.getBlock(blockHash);
+    const blockNumber = signedBlock.block.header.number.toNumber();
+    const chainName = (await apiPromise.rpc.system.chain()).toString();
+    const finalDbPath = dbPath ?? `db-${chainName}-${blockNumber}.sqlite`;
+    const api = new _chopstickscore.Api(provider);
+    const db = new _chopsticksdb.SqliteDatabase(finalDbPath);
+    logger.info(`Storages will be saved at ${finalDbPath}, use '--db=${finalDbPath} --block=${blockNumber}' to apply it later on`);
+    for (const prefix of prefixes){
+        let startKey = '0x';
+        let hasMorePages = true;
+        while(hasMorePages){
+            logger.debug({
+                prefix,
+                startKey
+            }, 'fetching keys');
+            const keysPage = await api.getKeysPaged(prefix, BATCH_SIZE, startKey, blockHash);
+            logger.debug({
+                prefix,
+                startKey
+            }, `fetched ${keysPage.length} keys`);
+            if (!keysPage.length) break;
+            startKey = keysPage[keysPage.length - 1];
+            if (!keysPage || keysPage.length < BATCH_SIZE) {
+                hasMorePages = false;
+            }
+            logger.debug({
+                prefix
+            }, 'fetching storages');
+            const storages = await api.getStorageBatch(prefix, keysPage, blockHash);
+            logger.debug({
+                prefix
+            }, `fetched ${storages.length} storages`);
+            const keyValueEntries = storages.map(([key, value])=>({
+                    blockHash,
+                    key,
+                    value
+                }));
+            await db.saveStorageBatch(keyValueEntries);
+            logger.debug({
+                prefix
+            }, `saved ${storages.length} storages ✅`);
+        }
+    }
+    logger.info(`Storages are saved at ${finalDbPath}, use '--db=${finalDbPath} --block=${blockNumber}' to apply it`);
+};
+const startFetchStorageWorker = async (options)=>{
+    if (!options.config) return null;
+    const worker = new _nodeworker_threads.default.Worker(new URL('./fetch-storages-worker.js', require("url").pathToFileURL(__filename).toString()), {
+        name: 'fetch-storages-worker'
+    });
+    const workerApi = (0, _comlink.wrap)((0, _nodeadapter.default)(worker));
+    workerApi.startFetch(options);
+    const terminate = async ()=>{
+        workerApi[_comlink.releaseProxy]();
+        await worker.terminate();
+    };
+    return {
+        worker: workerApi,
+        terminate
+    };
+};

package/dist/cjs/utils/index.d.ts

@@ -2,3 +2,4 @@ export * from './decoder.js';
 export * from './generate-html-diff.js';
 export * from './open-html.js';
 export * from './override.js';
+export * from './fetch-storages.js';

package/dist/cjs/utils/index.js

@@ -6,6 +6,7 @@ _export_star(require("./decoder.js"), exports);
 _export_star(require("./generate-html-diff.js"), exports);
 _export_star(require("./open-html.js"), exports);
 _export_star(require("./override.js"), exports);
+_export_star(require("./fetch-storages.js"), exports);
 function _export_star(from, to) {
     Object.keys(from).forEach(function(k) {
         if (k !== "default" && !Object.prototype.hasOwnProperty.call(to, k)) {

package/dist/esm/cli.js

@@ -4,10 +4,13 @@ import _ from 'lodash';
 import yargs from 'yargs';
 import { connectParachains, connectVertical, environment } from '@acala-network/chopsticks-core';
 import { configSchema, fetchConfig, getYargsOptions } from './schema/index.js';
-import { pluginExtendCli } from './plugins/index.js';
+import { loadRpcMethodsByScripts, pluginExtendCli } from './plugins/index.js';
 import { setupWithServer } from './index.js';
 dotenvConfig();
 const processArgv = async (argv)=>{
+    if (argv.unsafeRpcMethods) {
+        await loadRpcMethodsByScripts(argv.unsafeRpcMethods);
+    }
     if (argv.config) {
         Object.assign(argv, _.defaults(argv, await fetchConfig(argv.config)));
     }
@@ -43,7 +46,7 @@ const commands = yargs(hideBin(process.argv)).scriptName('chopsticks').middlewar
             await connectVertical(relaychain, parachain);
         }
     }
-}).strict().help().alias('help', 'h').alias('version', 'v').alias('config', 'c').alias('endpoint', 'e').alias('port', 'p').alias('block', 'b').alias('import-storage', 's').alias('wasm-override', 'w').usage('Usage: $0 <command> [options]').example('$0', '-c acala');
+}).strict().help().alias('help', 'h').alias('version', 'v').alias('config', 'c').alias('endpoint', 'e').alias('port', 'p').alias('block', 'b').alias('unsafe-rpc-methods', 'ur').alias('import-storage', 's').alias('wasm-override', 'w').usage('Usage: $0 <command> [options]').example('$0', '-c acala');
 if (!environment.DISABLE_PLUGINS) {
     pluginExtendCli(commands.config('config', 'Path to config file with default options', ()=>({}))).then(()=>commands.parse());
 } else {

package/dist/esm/context.d.ts

@@ -4,4 +4,15 @@ import { Config } from './schema/index.js';
 export declare const genesisFromUrl: (url: string) => Promise<GenesisProvider>;
 export declare const setupContext: (argv: Config, overrideParent?: boolean) => Promise<{
     chain: import("@acala-network/chopsticks-core").Blockchain;
+    fetchStorageWorker: {
+        worker: import("comlink").Remote<{
+            startFetch: (options: {
+                block?: string | number | null | undefined;
+                endpoint?: string | string[] | undefined;
+                dbPath?: string | undefined;
+                config: import("./utils/fetch-storages.js").FetchStorageConfig;
+            }) => Promise<void>;
+        }>;
+        terminate: () => Promise<void>;
+    } | null;
 }>;

package/dist/esm/context.js

@@ -1,7 +1,9 @@
 import './utils/tunnel.js';
 import { GenesisProvider, defaultLogger, isUrl, setup, timeTravel } from '@acala-network/chopsticks-core';
 import { SqliteDatabase } from '@acala-network/chopsticks-db';
+import { apiFetching } from './logger.js';
 import { overrideStorage, overrideWasm } from './utils/override.js';
+import { startFetchStorageWorker } from './utils/fetch-storages.js';
 import axios from 'axios';
 const logger = defaultLogger.child({
     name: 'setup-context'
@@ -41,7 +43,10 @@ export const setupContext = async (argv, overrideParent = false)=>{
         registeredTypes: argv['registered-types'],
         offchainWorker: argv['offchain-worker'],
         maxMemoryBlockCount: argv['max-memory-block-count'],
-        processQueuedMessages: argv['process-queued-messages']
+        processQueuedMessages: argv['process-queued-messages'],
+        hooks: {
+            apiFetching
+        }
     });
     // load block from db
     if (chain.db) {
@@ -77,7 +82,14 @@ export const setupContext = async (argv, overrideParent = false)=>{
     // added that have storage imports
     await overrideWasm(chain, argv['wasm-override'], at);
     await overrideStorage(chain, argv['import-storage'], at);
+    const fetchStorageWorker = await startFetchStorageWorker({
+        config: argv['prefetch-storages'],
+        dbPath: argv.db,
+        block: argv.block,
+        endpoint: argv.endpoint
+    });
     return {
-        chain
+        chain,
+        fetchStorageWorker
     };
 };

package/dist/esm/logger.d.ts

@@ -1 +1,4 @@
+import _ from 'lodash';
 export { defaultLogger, truncate } from '@acala-network/chopsticks-core';
+export declare const spinnerFrames: string[];
+export declare const apiFetching: _.DebouncedFunc<() => void>;

package/dist/esm/logger.js

@@ -1 +1,35 @@
+import _ from 'lodash';
 export { defaultLogger, truncate } from '@acala-network/chopsticks-core';
+const showProgress = process.stdout.isTTY && !process.env['CI'] && !process.env['TEST'];
+export const spinnerFrames = process.platform === 'win32' ? [
+    '-',
+    '\\',
+    '|',
+    '/'
+] : [
+    '⠋',
+    '⠙',
+    '⠹',
+    '⠸',
+    '⠼',
+    '⠴',
+    '⠦',
+    '⠧',
+    '⠇',
+    '⠏'
+];
+let index = 0;
+// clear to the right from cursor
+const clearStatus = _.debounce(()=>process.stdout.clearLine(1), 500, {
+    trailing: true
+});
+export const apiFetching = _.throttle(()=>{
+    if (!showProgress) return;
+    // print ` ⠋ Fetching|` and move cursor at position 0 of the line `| ⠋ Fetching`
+    process.stdout.write(` ${spinnerFrames[index++]} Fetching`);
+    process.stdout.cursorTo(0);
+    index = ++index % spinnerFrames.length;
+    clearStatus();
+}, 50, {
+    leading: true
+});

package/dist/esm/plugins/dry-run/index.d.ts

@@ -83,6 +83,7 @@ export declare const dryRunSchema: z.ZodObject<{
     'offchain-worker': z.ZodOptional<z.ZodBoolean>;
     resume: z.ZodOptional<z.ZodUnion<[z.ZodIntersection<z.ZodString, z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>>, z.ZodNumber, z.ZodBoolean]>>;
     'process-queued-messages': z.ZodOptional<z.ZodBoolean>;
+    'prefetch-storages': z.ZodOptional<z.ZodAny>;
 }, "strip", z.ZodTypeAny, {
     port: number;
     'build-block-mode': import("@acala-network/chopsticks-core").BuildBlockMode;
@@ -121,6 +122,7 @@ export declare const dryRunSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }, {
     extrinsic?: string | undefined;
     address?: string | undefined;
@@ -159,6 +161,7 @@ export declare const dryRunSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }>;
 export type DryRunSchemaType = z.infer<typeof dryRunSchema>;
 export * from './cli.js';

package/dist/esm/plugins/dry-run/rpc.d.ts

@@ -7,11 +7,11 @@ declare const schema: z.ZodObject<{
         call: z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>;
         address: z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>;
     }, "strip", z.ZodTypeAny, {
-        address: `0x${string}`;
         call: `0x${string}`;
-    }, {
         address: `0x${string}`;
+    }, {
         call: `0x${string}`;
+        address: `0x${string}`;
     }>]>>;
     hrmp: z.ZodOptional<z.ZodRecord<z.ZodEffects<z.ZodString, number, string>, z.ZodArray<z.ZodObject<{
         sentAt: z.ZodNumber;
@@ -39,8 +39,8 @@ declare const schema: z.ZodObject<{
     raw?: boolean | undefined;
     html?: boolean | undefined;
     extrinsic?: `0x${string}` | {
-        address: `0x${string}`;
         call: `0x${string}`;
+        address: `0x${string}`;
     } | undefined;
     hrmp?: Record<number, {
         data: `0x${string}`;
@@ -56,8 +56,8 @@ declare const schema: z.ZodObject<{
     raw?: boolean | undefined;
     html?: boolean | undefined;
     extrinsic?: `0x${string}` | {
-        address: `0x${string}`;
         call: `0x${string}`;
+        address: `0x${string}`;
     } | undefined;
     hrmp?: Record<string, {
         data: `0x${string}`;

package/dist/esm/plugins/fetch-storage/cli.d.ts

@@ -0,0 +1,2 @@
+import type { Argv } from 'yargs';
+export declare const cli: (y: Argv) => void;

package/dist/esm/plugins/fetch-storage/cli.js

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+import _ from 'lodash';
+import { configSchema, getYargsOptions } from '../../schema/index.js';
+import { fetchStorages } from '../../utils/fetch-storages.js';
+const schema = z.object(_.pick(configSchema.shape, [
+    'endpoint',
+    'block',
+    'db'
+]));
+export const cli = (y)=>{
+    y.command({
+        command: 'fetch-storages [items..]',
+        aliases: [
+            'fetch-storage'
+        ],
+        describe: 'Fetch and save storages',
+        builder: (yargs)=>yargs.options(getYargsOptions(schema.shape)),
+        handler: async (argv)=>{
+            const config = schema.parse(argv);
+            if (!argv.items) throw new Error('fetch-storages items are required');
+            try {
+                await fetchStorages({
+                    block: config.block,
+                    endpoint: config.endpoint,
+                    dbPath: config.db,
+                    config: argv.items
+                });
+                process.exit(0);
+            } catch (e) {
+                process.exit(1);
+            }
+        }
+    });
+};

package/dist/esm/plugins/fetch-storage/index.d.ts

@@ -0,0 +1 @@
+export * from './cli.js';

package/dist/esm/plugins/fetch-storage/index.js

@@ -0,0 +1 @@
+export * from './cli.js';

package/dist/esm/plugins/index.d.ts

@@ -2,5 +2,7 @@ import { Handlers } from '@acala-network/chopsticks-core';
 import type { Argv } from 'yargs';
 export declare const rpcPluginHandlers: Handlers;
 export declare const rpcPluginMethods: string[];
-export declare const loadRpcPlugin: (method: string) => Promise<any>;
+export declare const loadRpcMethodsByScripts: (path: string) => Promise<void>;
+export declare const getRpcExtensionMethods: () => string[];
+export declare const loadRpcExtensionMethod: (method: string) => Promise<any>;
 export declare const pluginExtendCli: (y: Argv) => Promise<void>;

package/dist/esm/plugins/index.js

@@ -1,7 +1,8 @@
 import { environment } from '@acala-network/chopsticks-core';
-import { lstatSync, readdirSync } from 'fs';
+import { lstatSync, readFileSync, readdirSync } from 'fs';
 import _ from 'lodash';
 import { defaultLogger } from '../logger.js';
+import { resolve } from 'path';
 const logger = defaultLogger.child({
     name: 'plugin'
 });
@@ -10,7 +11,7 @@ export const rpcPluginHandlers = {};
 const plugins = readdirSync(new URL('.', import.meta.url)).filter((file)=>lstatSync(new URL(file, import.meta.url)).isDirectory());
 // find all rpc methods
 export const rpcPluginMethods = plugins.filter((name)=>readdirSync(new URL(name, import.meta.url)).some((file)=>file.startsWith('rpc'))).map((name)=>`dev_${_.camelCase(name)}`);
-export const loadRpcPlugin = async (method)=>{
+const loadRpcPlugin = async (method)=>{
     if (environment.DISABLE_PLUGINS) {
         return undefined;
     }
@@ -24,6 +25,28 @@ export const loadRpcPlugin = async (method)=>{
     logger.debug(`Registered plugin ${plugin} RPC`);
     return rpc;
 };
+// store the loaded methods by cli
+let rpcScriptMethods = {};
+// use cli to load rpc methods of external scripts
+export const loadRpcMethodsByScripts = async (path)=>{
+    try {
+        const scriptContent = readFileSync(resolve(path), 'utf8');
+        rpcScriptMethods = new Function(scriptContent)();
+        logger.info(`${Object.keys(rpcScriptMethods).length} extension rpc methods loaded from ${path}`);
+    } catch (error) {
+        console.log('Failed to load rpc extension methods');
+    }
+};
+export const getRpcExtensionMethods = ()=>{
+    return [
+        ...Object.keys(rpcScriptMethods),
+        ...rpcPluginMethods
+    ];
+};
+export const loadRpcExtensionMethod = async (method)=>{
+    if (rpcScriptMethods[method]) return rpcScriptMethods[method];
+    return loadRpcPlugin(method);
+};
 export const pluginExtendCli = async (y)=>{
     for (const plugin of plugins){
         const location = new URL(`${plugin}/index.js`, import.meta.url);

package/dist/esm/rpc/index.js

@@ -1,5 +1,5 @@
 import { ResponseError, allHandlers as coreHandlers, defaultLogger } from '@acala-network/chopsticks-core';
-import { loadRpcPlugin, rpcPluginMethods } from '../plugins/index.js';
+import { getRpcExtensionMethods, loadRpcExtensionMethod } from '../plugins/index.js';
 const rpcLogger = defaultLogger.child({
     name: 'rpc'
 });
@@ -9,15 +9,15 @@ const allHandlers = {
             version: 1,
             methods: [
                 ...Object.keys(allHandlers),
-                ...rpcPluginMethods
+                ...getRpcExtensionMethods()
             ].sort()
         })
 };
 const getHandler = async (method)=>{
     const handler = allHandlers[method];
     if (!handler) {
-        // no handler for this method, check if it's a plugin
-        return loadRpcPlugin(method);
+        // no handler for this method, check if it's a plugin or a script loaded
+        return loadRpcExtensionMethod(method);
     }
     return handler;
 };

package/dist/esm/schema/index.d.ts

@@ -79,6 +79,7 @@ export declare const configSchema: z.ZodObject<{
     'offchain-worker': z.ZodOptional<z.ZodBoolean>;
     resume: z.ZodOptional<z.ZodUnion<[z.ZodIntersection<z.ZodString, z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>>, z.ZodNumber, z.ZodBoolean]>>;
     'process-queued-messages': z.ZodOptional<z.ZodBoolean>;
+    'prefetch-storages': z.ZodOptional<z.ZodAny>;
 }, "strip", ZodTypeAny, {
     port: number;
     'build-block-mode': BuildBlockMode;
@@ -110,6 +111,7 @@ export declare const configSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }, {
     port?: number | undefined;
     endpoint?: string | string[] | undefined;
@@ -141,6 +143,7 @@ export declare const configSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }>;
 export type Config = z.infer<typeof configSchema>;
 export declare const getYargsOptions: (zodShape: ZodRawShape) => {

package/dist/esm/schema/index.js

@@ -19,7 +19,7 @@ export const configSchema = z.object({
     }).optional(),
     block: z.union([
         z.string(),
-        z.number().max(Number.MAX_SAFE_INTEGER, 'Number is too big, please make it a string if you are uing a hex string'),
+        z.number().max(Number.MAX_SAFE_INTEGER, 'Number is too big, please make it a string if you are using a hex string'),
         z.null()
     ], {
         description: 'Block hash or block number. Default to latest block'
@@ -62,6 +62,9 @@ export const configSchema = z.object({
     }).optional(),
     'process-queued-messages': z.boolean({
         description: 'Produce extra block when queued messages are detected. Default to true. Set to false to disable it.'
+    }).optional(),
+    'prefetch-storages': z.any({
+        description: 'Storage key prefixes config for fetching storage, useful for testing big migrations, see README for examples'
     }).optional()
 });
 const getZodType = (option)=>{

package/dist/esm/setup-with-server.d.ts

@@ -3,4 +3,15 @@ export declare const setupWithServer: (argv: Config) => Promise<{
     listenPort: number;
     close(): Promise<void>;
     chain: import("@acala-network/chopsticks-core").Blockchain;
+    fetchStorageWorker: {
+        worker: import("comlink").Remote<{
+            startFetch: (options: {
+                block?: string | number | null | undefined;
+                endpoint?: string | string[] | undefined;
+                dbPath?: string | undefined;
+                config: import("./utils/fetch-storages.js").FetchStorageConfig;
+            }) => Promise<void>;
+        }>;
+        terminate: () => Promise<void>;
+    } | null;
 }>;

package/dist/esm/setup-with-server.js

@@ -11,6 +11,7 @@ export const setupWithServer = async (argv)=>{
         listenPort,
         async close () {
             await context.chain.close();
+            await context.fetchStorageWorker?.terminate();
             await close();
         }
     };

package/dist/esm/utils/fetch-storages-worker.js

@@ -0,0 +1,12 @@
+import * as Comlink from 'comlink';
+import { parentPort } from 'node:worker_threads';
+import nodeEndpoint from 'comlink/dist/umd/node-adapter.js';
+import { fetchStorages } from './fetch-storages.js';
+const api = {
+    startFetch: async ({ ...options })=>{
+        await fetchStorages({
+            ...options
+        });
+    }
+};
+Comlink.expose(api, nodeEndpoint(parentPort));

package/dist/esm/utils/fetch-storages.d.ts

@@ -0,0 +1,26 @@
+import { ApiPromise } from '@polkadot/api';
+import { HexString } from '@polkadot/util/types';
+export declare const logger: import("pino").default.Logger<never>;
+type FetchStorageConfigItem = HexString | string | Record<string, string | Record<string, any[]> | Record<string, any>[] | (string | any)[]>;
+export type FetchStorageConfig = FetchStorageConfigItem[];
+/**
+ * Convert fetch-storage configs to prefixes for fetching.
+ */
+export declare const getPrefixesFromConfig: (config: FetchStorageConfig, api: ApiPromise) => Promise<string[]>;
+type FetchStoragesParams = {
+    block?: number | string | null;
+    endpoint?: string | string[];
+    dbPath?: string;
+    config: FetchStorageConfig;
+};
+/**
+ * Fetch storages and save in a local db
+ */
+export declare const fetchStorages: ({ block, endpoint, dbPath, config }: FetchStoragesParams) => Promise<void>;
+export declare const startFetchStorageWorker: (options: FetchStoragesParams) => Promise<{
+    worker: import("comlink").Remote<{
+        startFetch: (options: FetchStoragesParams) => Promise<void>;
+    }>;
+    terminate: () => Promise<void>;
+} | null>;
+export {};