@acala-network/chopsticks 0.10.2 → 0.12.0

package/dist/cjs/cli.js

@@ -17,6 +17,9 @@ function _interop_require_default(obj) {
 }
 (0, _dotenv.config)();
 const processArgv = async (argv)=>{
+    if (argv.unsafeRpcMethods) {
+        await (0, _index1.loadRpcMethodsByScripts)(argv.unsafeRpcMethods);
+    }
     if (argv.config) {
         Object.assign(argv, _lodash.default.defaults(argv, await (0, _index.fetchConfig)(argv.config)));
     }
@@ -52,7 +55,7 @@ const commands = (0, _yargs.default)((0, _helpers.hideBin)(process.argv)).script
             await (0, _chopstickscore.connectVertical)(relaychain, parachain);
         }
     }
-}).strict().help().alias('help', 'h').alias('version', 'v').alias('config', 'c').alias('endpoint', 'e').alias('port', 'p').alias('block', 'b').alias('import-storage', 's').alias('wasm-override', 'w').usage('Usage: $0 <command> [options]').example('$0', '-c acala');
+}).strict().help().alias('help', 'h').alias('version', 'v').alias('config', 'c').alias('endpoint', 'e').alias('port', 'p').alias('block', 'b').alias('unsafe-rpc-methods', 'ur').alias('import-storage', 's').alias('wasm-override', 'w').usage('Usage: $0 <command> [options]').example('$0', '-c acala');
 if (!_chopstickscore.environment.DISABLE_PLUGINS) {
     (0, _index1.pluginExtendCli)(commands.config('config', 'Path to config file with default options', ()=>({}))).then(()=>commands.parse());
 } else {

package/dist/cjs/context.d.ts

@@ -4,4 +4,15 @@ import { Config } from './schema/index.js';
 export declare const genesisFromUrl: (url: string) => Promise<GenesisProvider>;
 export declare const setupContext: (argv: Config, overrideParent?: boolean) => Promise<{
     chain: import("@acala-network/chopsticks-core").Blockchain;
+    fetchStorageWorker: {
+        worker: import("comlink").Remote<{
+            startFetch: (options: {
+                block?: string | number | null | undefined;
+                endpoint?: string | string[] | undefined;
+                dbPath?: string | undefined;
+                config: import("./utils/fetch-storages.js").FetchStorageConfig;
+            }) => Promise<void>;
+        }>;
+        terminate: () => Promise<void>;
+    } | null;
 }>;

package/dist/cjs/context.js

@@ -19,7 +19,9 @@ _export(exports, {
 require("./utils/tunnel.js");
 const _chopstickscore = require("@acala-network/chopsticks-core");
 const _chopsticksdb = require("@acala-network/chopsticks-db");
+const _logger = require("./logger.js");
 const _override = require("./utils/override.js");
+const _fetchstorages = require("./utils/fetch-storages.js");
 const _axios = /*#__PURE__*/ _interop_require_default(require("axios"));
 function _interop_require_default(obj) {
     return obj && obj.__esModule ? obj : {
@@ -105,7 +107,10 @@ const setupContext = async (argv, overrideParent = false)=>{
         registeredTypes: argv['registered-types'],
         offchainWorker: argv['offchain-worker'],
         maxMemoryBlockCount: argv['max-memory-block-count'],
-        processQueuedMessages: argv['process-queued-messages']
+        processQueuedMessages: argv['process-queued-messages'],
+        hooks: {
+            apiFetching: _logger.apiFetching
+        }
     });
     // load block from db
     if (chain.db) {
@@ -141,7 +146,14 @@ const setupContext = async (argv, overrideParent = false)=>{
     // added that have storage imports
     await (0, _override.overrideWasm)(chain, argv['wasm-override'], at);
     await (0, _override.overrideStorage)(chain, argv['import-storage'], at);
+    const fetchStorageWorker = await (0, _fetchstorages.startFetchStorageWorker)({
+        config: argv['prefetch-storages'],
+        dbPath: argv.db,
+        block: argv.block,
+        endpoint: argv.endpoint
+    });
     return {
-        chain
+        chain,
+        fetchStorageWorker
     };
 };

package/dist/cjs/logger.d.ts

@@ -1 +1,4 @@
+import _ from 'lodash';
 export { defaultLogger, truncate } from '@acala-network/chopsticks-core';
+export declare const spinnerFrames: string[];
+export declare const apiFetching: _.DebouncedFunc<() => void>;

package/dist/cjs/logger.js

@@ -9,11 +9,56 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    apiFetching: function() {
+        return apiFetching;
+    },
     defaultLogger: function() {
         return _chopstickscore.defaultLogger;
     },
+    spinnerFrames: function() {
+        return spinnerFrames;
+    },
     truncate: function() {
         return _chopstickscore.truncate;
     }
 });
+const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
 const _chopstickscore = require("@acala-network/chopsticks-core");
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+const showProgress = process.stdout.isTTY && !process.env['CI'] && !process.env['TEST'];
+const spinnerFrames = process.platform === 'win32' ? [
+    '-',
+    '\\',
+    '|',
+    '/'
+] : [
+    '⠋',
+    '⠙',
+    '⠹',
+    '⠸',
+    '⠼',
+    '⠴',
+    '⠦',
+    '⠧',
+    '⠇',
+    '⠏'
+];
+let index = 0;
+// clear to the right from cursor
+const clearStatus = _lodash.default.debounce(()=>process.stdout.clearLine(1), 500, {
+    trailing: true
+});
+const apiFetching = _lodash.default.throttle(()=>{
+    if (!showProgress) return;
+    // print ` ⠋ Fetching|` and move cursor at position 0 of the line `| ⠋ Fetching`
+    process.stdout.write(` ${spinnerFrames[index++]} Fetching`);
+    process.stdout.cursorTo(0);
+    index = ++index % spinnerFrames.length;
+    clearStatus();
+}, 50, {
+    leading: true
+});

package/dist/cjs/plugins/dry-run/index.d.ts

@@ -83,6 +83,7 @@ export declare const dryRunSchema: z.ZodObject<{
     'offchain-worker': z.ZodOptional<z.ZodBoolean>;
     resume: z.ZodOptional<z.ZodUnion<[z.ZodIntersection<z.ZodString, z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>>, z.ZodNumber, z.ZodBoolean]>>;
     'process-queued-messages': z.ZodOptional<z.ZodBoolean>;
+    'prefetch-storages': z.ZodOptional<z.ZodAny>;
 }, "strip", z.ZodTypeAny, {
     port: number;
     'build-block-mode': import("@acala-network/chopsticks-core").BuildBlockMode;
@@ -121,6 +122,7 @@ export declare const dryRunSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }, {
     extrinsic?: string | undefined;
     address?: string | undefined;
@@ -159,6 +161,7 @@ export declare const dryRunSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }>;
 export type DryRunSchemaType = z.infer<typeof dryRunSchema>;
 export * from './cli.js';

package/dist/cjs/plugins/dry-run/rpc.d.ts

@@ -7,11 +7,11 @@ declare const schema: z.ZodObject<{
         call: z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>;
         address: z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>;
     }, "strip", z.ZodTypeAny, {
-        address: `0x${string}`;
         call: `0x${string}`;
-    }, {
         address: `0x${string}`;
+    }, {
         call: `0x${string}`;
+        address: `0x${string}`;
     }>]>>;
     hrmp: z.ZodOptional<z.ZodRecord<z.ZodEffects<z.ZodString, number, string>, z.ZodArray<z.ZodObject<{
         sentAt: z.ZodNumber;
@@ -39,8 +39,8 @@ declare const schema: z.ZodObject<{
     raw?: boolean | undefined;
     html?: boolean | undefined;
     extrinsic?: `0x${string}` | {
-        address: `0x${string}`;
         call: `0x${string}`;
+        address: `0x${string}`;
     } | undefined;
     hrmp?: Record<number, {
         data: `0x${string}`;
@@ -56,8 +56,8 @@ declare const schema: z.ZodObject<{
     raw?: boolean | undefined;
     html?: boolean | undefined;
     extrinsic?: `0x${string}` | {
-        address: `0x${string}`;
         call: `0x${string}`;
+        address: `0x${string}`;
     } | undefined;
     hrmp?: Record<string, {
         data: `0x${string}`;

package/dist/cjs/plugins/fetch-storage/cli.d.ts

@@ -0,0 +1,2 @@
+import type { Argv } from 'yargs';
+export declare const cli: (y: Argv) => void;

package/dist/cjs/plugins/fetch-storage/cli.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "cli", {
+    enumerable: true,
+    get: function() {
+        return cli;
+    }
+});
+const _zod = require("zod");
+const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
+const _index = require("../../schema/index.js");
+const _fetchstorages = require("../../utils/fetch-storages.js");
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+const schema = _zod.z.object(_lodash.default.pick(_index.configSchema.shape, [
+    'endpoint',
+    'block',
+    'db'
+]));
+const cli = (y)=>{
+    y.command({
+        command: 'fetch-storages [items..]',
+        aliases: [
+            'fetch-storage'
+        ],
+        describe: 'Fetch and save storages',
+        builder: (yargs)=>yargs.options((0, _index.getYargsOptions)(schema.shape)),
+        handler: async (argv)=>{
+            const config = schema.parse(argv);
+            if (!argv.items) throw new Error('fetch-storages items are required');
+            try {
+                await (0, _fetchstorages.fetchStorages)({
+                    block: config.block,
+                    endpoint: config.endpoint,
+                    dbPath: config.db,
+                    config: argv.items
+                });
+                process.exit(0);
+            } catch (e) {
+                process.exit(1);
+            }
+        }
+    });
+};

package/dist/cjs/plugins/fetch-storage/index.d.ts

@@ -0,0 +1 @@
+export * from './cli.js';

package/dist/cjs/plugins/fetch-storage/index.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+_export_star(require("./cli.js"), exports);
+function _export_star(from, to) {
+    Object.keys(from).forEach(function(k) {
+        if (k !== "default" && !Object.prototype.hasOwnProperty.call(to, k)) {
+            Object.defineProperty(to, k, {
+                enumerable: true,
+                get: function() {
+                    return from[k];
+                }
+            });
+        }
+    });
+    return from;
+}

package/dist/cjs/plugins/index.d.ts

@@ -2,5 +2,7 @@ import { Handlers } from '@acala-network/chopsticks-core';
 import type { Argv } from 'yargs';
 export declare const rpcPluginHandlers: Handlers;
 export declare const rpcPluginMethods: string[];
-export declare const loadRpcPlugin: (method: string) => Promise<any>;
+export declare const loadRpcMethodsByScripts: (path: string) => Promise<void>;
+export declare const getRpcExtensionMethods: () => string[];
+export declare const loadRpcExtensionMethod: (method: string) => Promise<any>;
 export declare const pluginExtendCli: (y: Argv) => Promise<void>;

package/dist/cjs/plugins/index.js

@@ -9,8 +9,14 @@ function _export(target, all) {
     });
 }
 _export(exports, {
-    loadRpcPlugin: function() {
-        return loadRpcPlugin;
+    getRpcExtensionMethods: function() {
+        return getRpcExtensionMethods;
+    },
+    loadRpcExtensionMethod: function() {
+        return loadRpcExtensionMethod;
+    },
+    loadRpcMethodsByScripts: function() {
+        return loadRpcMethodsByScripts;
     },
     pluginExtendCli: function() {
         return pluginExtendCli;
@@ -26,6 +32,7 @@ const _chopstickscore = require("@acala-network/chopsticks-core");
 const _fs = require("fs");
 const _lodash = /*#__PURE__*/ _interop_require_default(require("lodash"));
 const _logger = require("../logger.js");
+const _path = require("path");
 function _interop_require_default(obj) {
     return obj && obj.__esModule ? obj : {
         default: obj
@@ -93,6 +100,27 @@ const loadRpcPlugin = async (method)=>{
     logger.debug(`Registered plugin ${plugin} RPC`);
     return rpc;
 };
+// store the loaded methods by cli
+let rpcScriptMethods = {};
+const loadRpcMethodsByScripts = async (path)=>{
+    try {
+        const scriptContent = (0, _fs.readFileSync)((0, _path.resolve)(path), 'utf8');
+        rpcScriptMethods = new Function(scriptContent)();
+        logger.info(`${Object.keys(rpcScriptMethods).length} extension rpc methods loaded from ${path}`);
+    } catch (error) {
+        console.log('Failed to load rpc extension methods');
+    }
+};
+const getRpcExtensionMethods = ()=>{
+    return [
+        ...Object.keys(rpcScriptMethods),
+        ...rpcPluginMethods
+    ];
+};
+const loadRpcExtensionMethod = async (method)=>{
+    if (rpcScriptMethods[method]) return rpcScriptMethods[method];
+    return loadRpcPlugin(method);
+};
 const pluginExtendCli = async (y)=>{
     for (const plugin of plugins){
         const location = new URL(`${plugin}/index.js`, require("url").pathToFileURL(__filename).toString());

package/dist/cjs/rpc/index.js

@@ -19,15 +19,15 @@ const allHandlers = {
             version: 1,
             methods: [
                 ...Object.keys(allHandlers),
-                ..._index.rpcPluginMethods
+                ...(0, _index.getRpcExtensionMethods)()
             ].sort()
         })
 };
 const getHandler = async (method)=>{
     const handler = allHandlers[method];
     if (!handler) {
-        // no handler for this method, check if it's a plugin
-        return (0, _index.loadRpcPlugin)(method);
+        // no handler for this method, check if it's a plugin or a script loaded
+        return (0, _index.loadRpcExtensionMethod)(method);
     }
     return handler;
 };

package/dist/cjs/schema/index.d.ts

@@ -79,6 +79,7 @@ export declare const configSchema: z.ZodObject<{
     'offchain-worker': z.ZodOptional<z.ZodBoolean>;
     resume: z.ZodOptional<z.ZodUnion<[z.ZodIntersection<z.ZodString, z.ZodType<`0x${string}`, z.ZodTypeDef, `0x${string}`>>, z.ZodNumber, z.ZodBoolean]>>;
     'process-queued-messages': z.ZodOptional<z.ZodBoolean>;
+    'prefetch-storages': z.ZodOptional<z.ZodAny>;
 }, "strip", ZodTypeAny, {
     port: number;
     'build-block-mode': BuildBlockMode;
@@ -110,6 +111,7 @@ export declare const configSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }, {
     port?: number | undefined;
     endpoint?: string | string[] | undefined;
@@ -141,6 +143,7 @@ export declare const configSchema: z.ZodObject<{
     'offchain-worker'?: boolean | undefined;
     resume?: number | boolean | `0x${string}` | undefined;
     'process-queued-messages'?: boolean | undefined;
+    'prefetch-storages'?: any;
 }>;
 export type Config = z.infer<typeof configSchema>;
 export declare const getYargsOptions: (zodShape: ZodRawShape) => {

package/dist/cjs/schema/index.js

@@ -51,7 +51,7 @@ const configSchema = _zod.z.object({
     }).optional(),
     block: _zod.z.union([
         _zod.z.string(),
-        _zod.z.number().max(Number.MAX_SAFE_INTEGER, 'Number is too big, please make it a string if you are uing a hex string'),
+        _zod.z.number().max(Number.MAX_SAFE_INTEGER, 'Number is too big, please make it a string if you are using a hex string'),
         _zod.z.null()
     ], {
         description: 'Block hash or block number. Default to latest block'
@@ -94,6 +94,9 @@ const configSchema = _zod.z.object({
     }).optional(),
     'process-queued-messages': _zod.z.boolean({
         description: 'Produce extra block when queued messages are detected. Default to true. Set to false to disable it.'
+    }).optional(),
+    'prefetch-storages': _zod.z.any({
+        description: 'Storage key prefixes config for fetching storage, useful for testing big migrations, see README for examples'
     }).optional()
 });
 const getZodType = (option)=>{

package/dist/cjs/setup-with-server.d.ts

@@ -3,4 +3,15 @@ export declare const setupWithServer: (argv: Config) => Promise<{
     listenPort: number;
     close(): Promise<void>;
     chain: import("@acala-network/chopsticks-core").Blockchain;
+    fetchStorageWorker: {
+        worker: import("comlink").Remote<{
+            startFetch: (options: {
+                block?: string | number | null | undefined;
+                endpoint?: string | string[] | undefined;
+                dbPath?: string | undefined;
+                config: import("./utils/fetch-storages.js").FetchStorageConfig;
+            }) => Promise<void>;
+        }>;
+        terminate: () => Promise<void>;
+    } | null;
 }>;

package/dist/cjs/setup-with-server.js

@@ -21,6 +21,7 @@ const setupWithServer = async (argv)=>{
         listenPort,
         async close () {
             await context.chain.close();
+            await context.fetchStorageWorker?.terminate();
             await close();
         }
     };

package/dist/cjs/utils/fetch-storages-worker.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+const _comlink = /*#__PURE__*/ _interop_require_wildcard(require("comlink"));
+const _nodeworker_threads = require("node:worker_threads");
+const _nodeadapter = /*#__PURE__*/ _interop_require_default(require("comlink/dist/umd/node-adapter.js"));
+const _fetchstorages = require("./fetch-storages.js");
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+function _getRequireWildcardCache(nodeInterop) {
+    if (typeof WeakMap !== "function") return null;
+    var cacheBabelInterop = new WeakMap();
+    var cacheNodeInterop = new WeakMap();
+    return (_getRequireWildcardCache = function(nodeInterop) {
+        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
+    })(nodeInterop);
+}
+function _interop_require_wildcard(obj, nodeInterop) {
+    if (!nodeInterop && obj && obj.__esModule) {
+        return obj;
+    }
+    if (obj === null || typeof obj !== "object" && typeof obj !== "function") {
+        return {
+            default: obj
+        };
+    }
+    var cache = _getRequireWildcardCache(nodeInterop);
+    if (cache && cache.has(obj)) {
+        return cache.get(obj);
+    }
+    var newObj = {
+        __proto__: null
+    };
+    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
+    for(var key in obj){
+        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
+            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
+            if (desc && (desc.get || desc.set)) {
+                Object.defineProperty(newObj, key, desc);
+            } else {
+                newObj[key] = obj[key];
+            }
+        }
+    }
+    newObj.default = obj;
+    if (cache) {
+        cache.set(obj, newObj);
+    }
+    return newObj;
+}
+const api = {
+    startFetch: async ({ ...options })=>{
+        await (0, _fetchstorages.fetchStorages)({
+            ...options
+        });
+    }
+};
+_comlink.expose(api, (0, _nodeadapter.default)(_nodeworker_threads.parentPort));

package/dist/cjs/utils/fetch-storages.d.ts

@@ -0,0 +1,26 @@
+import { ApiPromise } from '@polkadot/api';
+import { HexString } from '@polkadot/util/types';
+export declare const logger: import("pino").default.Logger<never>;
+type FetchStorageConfigItem = HexString | string | Record<string, string | Record<string, any[]> | Record<string, any>[] | (string | any)[]>;
+export type FetchStorageConfig = FetchStorageConfigItem[];
+/**
+ * Convert fetch-storage configs to prefixes for fetching.
+ */
+export declare const getPrefixesFromConfig: (config: FetchStorageConfig, api: ApiPromise) => Promise<string[]>;
+type FetchStoragesParams = {
+    block?: number | string | null;
+    endpoint?: string | string[];
+    dbPath?: string;
+    config: FetchStorageConfig;
+};
+/**
+ * Fetch storages and save in a local db
+ */
+export declare const fetchStorages: ({ block, endpoint, dbPath, config }: FetchStoragesParams) => Promise<void>;
+export declare const startFetchStorageWorker: (options: FetchStoragesParams) => Promise<{
+    worker: import("comlink").Remote<{
+        startFetch: (options: FetchStoragesParams) => Promise<void>;
+    }>;
+    terminate: () => Promise<void>;
+} | null>;
+export {};