@acala-network/chopsticks 0.1.0

package/README.md

@@ -0,0 +1,52 @@
+# Chopsticks
+
+Create parallel reality of your Substrate network.
+
+## Install
+
+Make sure you have setup Rust environment (>= 1.64).
+
+- Clone repository with submodules ([smoldot](https://github.com/paritytech/smoldot))
+  - `git clone --recurse-submodules https://github.com/AcalaNetwork/chopsticks.git && cd chopsticks`
+- Install deps
+  - `yarn`
+- Build wasm
+  - `yarn build-wasm`
+
+## Run
+
+- Replay latest block
+  - `yarn start run-block --endpoint=wss://acala-rpc-2.aca-api.network/ws`
+  - This will replay the last block and print out the changed storages
+  - Use option `--output-path=<file_path>` to print out JSON file
+- Run a test node
+  - `yarn start dev --endpoint=wss://acala-rpc-2.aca-api.network/ws`
+  - You have a test node running at `ws://localhost:8000`
+  - You can use [Polkadot.js Apps](https://polkadot.js.org/apps/) to connect to this node
+  - Submit any transaction to produce a new block in the in parallel reality
+  - (Optional) Pre-define/override storage using option `--import-storage=storage.[json/yaml]`. See example storage below.
+  ```json5
+  // prettier-ignore
+  {
+    "Sudo": {
+      "Key": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"
+    },
+    "TechnicalCommittee": {
+      "Members": ["5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"]
+    },
+    "Tokens": {
+      "Accounts": [
+        [
+          ["5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", { "token": "KAR" }],
+          {
+            "free": 1000000000000000,
+          }
+        ]
+      ]
+    }
+  }
+  ```
+- Run Kusama fork
+  - Edit configs/kusama.yml if needed. (e.g. update the block number)
+  - `yarn start dev --config=configs/kusama.yml`
+	

package/chopsticks.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('./dist/index.js')

package/dist/api.d.ts

@@ -0,0 +1,43 @@
+import { ExtDef } from '@polkadot/types/extrinsic/signedExtensions/types';
+import { HexString } from '@polkadot/util/types';
+import { ProviderInterface } from '@polkadot/rpc-provider/types';
+type ChainProperties = {
+    ss58Format?: number;
+    tokenDecimals?: number[];
+    tokenSymbol?: string[];
+};
+type Header = {
+    parentHash: HexString;
+    number: HexString;
+    stateRoot: HexString;
+    extrinsicsRoot: HexString;
+    digest: {
+        logs: HexString[];
+    };
+};
+type SignedBlock = {
+    block: {
+        header: Header;
+        extrinsics: HexString[];
+    };
+    justifications?: HexString[];
+};
+export declare class Api {
+    #private;
+    readonly signedExtensions: ExtDef;
+    constructor(provider: ProviderInterface, signedExtensions?: ExtDef);
+    disconnect(): Promise<void>;
+    get isReady(): Promise<void>;
+    get chain(): Promise<string>;
+    get chainProperties(): Promise<ChainProperties>;
+    getSystemName(): Promise<string>;
+    getSystemProperties(): Promise<ChainProperties>;
+    getSystemChain(): Promise<string>;
+    getMetadata(hash?: string): Promise<string>;
+    getBlockHash(blockNumber?: number): Promise<string>;
+    getHeader(hash?: string): Promise<Header>;
+    getBlock(hash?: string): Promise<SignedBlock>;
+    getStorage(key: string, hash?: string): Promise<string>;
+    getKeysPaged(prefix: string, pageSize: number, startKey: string, hash?: string): Promise<string[]>;
+}
+export {};

package/dist/api.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Api = void 0;
+class Api {
+    #provider;
+    #isReady;
+    #chain;
+    #chainProperties;
+    signedExtensions;
+    constructor(provider, signedExtensions) {
+        this.#provider = provider;
+        this.signedExtensions = signedExtensions || {};
+        this.#isReady = new Promise((resolve, reject) => {
+            if (this.#provider.isConnected) {
+                setTimeout(resolve, 500);
+            }
+            else {
+                this.#provider.on('connected', () => {
+                    setTimeout(resolve, 500);
+                });
+            }
+            this.#provider.on('error', reject);
+        });
+        this.#provider.on('disconnected', () => {
+            // TODO: reconnect
+            console.warn('Api disconnected');
+        });
+        this.#chain = this.#isReady.then(() => this.getSystemChain());
+        this.#chainProperties = this.#isReady.then(() => this.getSystemProperties());
+        this.#provider.connect();
+    }
+    async disconnect() {
+        return this.#provider.disconnect();
+    }
+    get isReady() {
+        return this.#isReady;
+    }
+    get chain() {
+        return this.#chain;
+    }
+    get chainProperties() {
+        return this.#chainProperties;
+    }
+    async getSystemName() {
+        return this.#provider.send('system_name', []);
+    }
+    async getSystemProperties() {
+        return this.#provider.send('system_properties', []);
+    }
+    async getSystemChain() {
+        return this.#provider.send('system_chain', []);
+    }
+    async getMetadata(hash) {
+        return this.#provider.send('state_getMetadata', hash ? [hash] : []);
+    }
+    async getBlockHash(blockNumber) {
+        return this.#provider.send('chain_getBlockHash', Number.isInteger(blockNumber) ? [blockNumber] : []);
+    }
+    async getHeader(hash) {
+        return this.#provider.send('chain_getHeader', hash ? [hash] : []);
+    }
+    async getBlock(hash) {
+        return this.#provider.send('chain_getBlock', hash ? [hash] : []);
+    }
+    async getStorage(key, hash) {
+        if (hash) {
+            return this.#provider.send('state_getStorageAt', [key, hash]);
+        }
+        return this.#provider.send('state_getStorage', [key]);
+    }
+    async getKeysPaged(prefix, pageSize, startKey, hash) {
+        if (hash) {
+            return this.#provider.send('state_getKeysPagedAt', [prefix, pageSize, startKey, hash]);
+        }
+        return this.#provider.send('state_getKeysPaged', [prefix, pageSize, startKey]);
+    }
+}
+exports.Api = Api;

package/dist/blockchain/block.d.ts

@@ -0,0 +1,38 @@
+import { Header } from '@polkadot/types/interfaces';
+import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types';
+import { TypeRegistry } from '@polkadot/types';
+import type { HexString } from '@polkadot/util/types';
+import { Blockchain } from '.';
+import { StorageLayer, StorageLayerProvider } from './storage-layer';
+import { TaskResponseCall } from '../task';
+import type { RuntimeVersion } from '../executor';
+export declare class Block {
+    #private;
+    readonly number: number;
+    readonly hash: string;
+    constructor(chain: Blockchain, number: number, hash: string, parentBlock?: Block, block?: {
+        header: Header;
+        extrinsics: string[];
+        storage?: StorageLayerProvider;
+    });
+    get header(): Header | Promise<Header>;
+    get extrinsics(): string[] | Promise<string[]>;
+    get parentBlock(): undefined | Block | Promise<Block | undefined>;
+    get storage(): StorageLayerProvider;
+    get(key: string): Promise<string | undefined>;
+    getKeysPaged(options: {
+        prefix?: string;
+        startKey?: string;
+        pageSize: number;
+    }): Promise<string[]>;
+    pushStorageLayer(): StorageLayer;
+    popStorageLayer(): void;
+    storageDiff(): Promise<Record<string, string>>;
+    get wasm(): Promise<`0x${string}`>;
+    setWasm(wasm: HexString): void;
+    get registry(): Promise<TypeRegistry>;
+    get runtimeVersion(): Promise<RuntimeVersion>;
+    get metadata(): Promise<HexString>;
+    get meta(): Promise<DecoratedMeta>;
+    call(method: string, args: string): Promise<TaskResponseCall['Call']>;
+}

package/dist/blockchain/block.js

@@ -0,0 +1,178 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Block = void 0;
+const types_1 = require("@polkadot/types");
+const metadata_1 = require("@polkadot/types/metadata");
+const util_1 = require("@polkadot/types-known/util");
+const util_2 = require("@polkadot/util");
+const storage_layer_1 = require("./storage-layer");
+const shared_1 = require("../rpc/shared");
+const executor_1 = require("../executor");
+class Block {
+    number;
+    hash;
+    #chain;
+    #header;
+    #parentBlock;
+    #extrinsics;
+    #wasm;
+    #runtimeVersion;
+    #metadata;
+    #registry;
+    #meta;
+    #baseStorage;
+    #storages;
+    constructor(chain, number, hash, parentBlock, block) {
+        this.number = number;
+        this.hash = hash;
+        this.#chain = chain;
+        this.#parentBlock = parentBlock;
+        this.#header = block?.header;
+        this.#extrinsics = block?.extrinsics;
+        this.#baseStorage = block?.storage ?? new storage_layer_1.RemoteStorageLayer(chain.api, hash, chain.db);
+        this.#storages = [];
+        this.#registry = parentBlock?.registry;
+    }
+    get header() {
+        if (!this.#header) {
+            this.#header = Promise.all([this.registry, this.#chain.api.getHeader(this.hash)]).then(([registry, header]) => registry.createType('Header', header));
+        }
+        return this.#header;
+    }
+    get extrinsics() {
+        if (!this.#extrinsics) {
+            this.#extrinsics = this.#chain.api.getBlock(this.hash).then((b) => b.block.extrinsics);
+        }
+        return this.#extrinsics;
+    }
+    get parentBlock() {
+        if (this.number === 0) {
+            return undefined;
+        }
+        if (!this.#parentBlock) {
+            this.#parentBlock = Promise.resolve(this.header).then((h) => this.#chain.getBlock(h.parentHash.toHex()));
+        }
+        return this.#parentBlock;
+    }
+    get storage() {
+        return this.#storages[this.#storages.length - 1] ?? this.#baseStorage;
+    }
+    async get(key) {
+        const val = await this.storage.get(key, true);
+        switch (val) {
+            case "Deleted" /* StorageValueKind.Deleted */:
+                return undefined;
+            default:
+                return val;
+        }
+    }
+    async getKeysPaged(options) {
+        const layer = new storage_layer_1.StorageLayer(this.storage);
+        await layer.fold();
+        const prefix = options.prefix ?? '0x';
+        const startKey = options.startKey ?? prefix;
+        const pageSize = options.pageSize;
+        return layer.getKeysPaged(prefix, pageSize, startKey);
+    }
+    pushStorageLayer() {
+        const layer = new storage_layer_1.StorageLayer(this.storage);
+        this.#storages.push(layer);
+        return layer;
+    }
+    popStorageLayer() {
+        this.#storages.pop();
+    }
+    async storageDiff() {
+        const storage = {};
+        for (const layer of this.#storages) {
+            await layer.mergeInto(storage);
+        }
+        return storage;
+    }
+    get wasm() {
+        const getWasm = async () => {
+            const wasmKey = (0, util_2.stringToHex)(':code');
+            const wasm = await this.get(wasmKey);
+            if (!wasm) {
+                throw new Error('No wasm found');
+            }
+            return wasm;
+        };
+        if (!this.#wasm) {
+            this.#wasm = getWasm();
+        }
+        return this.#wasm;
+    }
+    setWasm(wasm) {
+        const wasmKey = (0, util_2.stringToHex)(':code');
+        this.pushStorageLayer().set(wasmKey, wasm);
+        this.#wasm = Promise.resolve(wasm);
+        this.#runtimeVersion = undefined;
+        this.#registry = undefined;
+        this.#meta = undefined;
+        this.#metadata = undefined;
+    }
+    get registry() {
+        if (!this.#registry) {
+            this.#registry = Promise.all([
+                this.metadata,
+                this.#chain.api.chainProperties,
+                this.#chain.api.chain,
+                this.runtimeVersion,
+            ]).then(([data, properties, chain, version]) => {
+                const registry = new types_1.TypeRegistry(this.hash);
+                registry.setChainProperties(registry.createType('ChainProperties', properties));
+                registry.register((0, util_1.getSpecTypes)(registry, chain, version.specName, version.specVersion));
+                registry.setHasher((0, util_1.getSpecHasher)(registry, chain, version.specName));
+                registry.setMetadata(new types_1.Metadata(registry, data), undefined, (0, util_2.objectSpread)({}, (0, util_1.getSpecExtensions)(registry, chain, version.specName), this.#chain.api.signedExtensions));
+                return registry;
+            });
+        }
+        return this.#registry;
+    }
+    get runtimeVersion() {
+        if (!this.#runtimeVersion) {
+            this.#runtimeVersion = this.wasm.then(executor_1.getRuntimeVersion);
+        }
+        return this.#runtimeVersion;
+    }
+    get metadata() {
+        if (!this.#metadata) {
+            this.#metadata = this.wasm.then(executor_1.getMetadata);
+        }
+        return this.#metadata;
+    }
+    get meta() {
+        if (!this.#meta) {
+            this.#meta = Promise.all([this.registry, this.metadata]).then(([registry, metadataStr]) => {
+                const metadata = new types_1.Metadata(registry, metadataStr);
+                return (0, metadata_1.expandMetadata)(registry, metadata);
+            });
+        }
+        return this.#meta;
+    }
+    async call(method, args) {
+        const wasm = await this.wasm;
+        const res = await new Promise((resolve, reject) => {
+            this.#chain.tasks.addAndRunTask({
+                Call: {
+                    blockHash: this.hash,
+                    wasm,
+                    calls: [[method, args]],
+                },
+            }, (r) => {
+                if ('Call' in r) {
+                    resolve(r.Call);
+                }
+                else if ('Error' in r) {
+                    reject(new shared_1.ResponseError(1, r.Error));
+                }
+                else {
+                    reject(new shared_1.ResponseError(1, 'Unexpected response'));
+                }
+            });
+        });
+        return res;
+    }
+}
+exports.Block = Block;

package/dist/blockchain/head-state.d.ts

@@ -0,0 +1,13 @@
+import { Block } from './block';
+export declare const randomId: () => string;
+export declare class HeadState {
+    #private;
+    constructor(head: Block);
+    subscribeHead(cb: (block: Block) => void): string;
+    unsubscribeHead(id: string): void;
+    subscribeStorage(keys: string[], cb: (block: Block, pairs: [string, string][]) => void): Promise<string>;
+    unsubscribeStorage(id: string): void;
+    subscrubeRuntimeVersion(cb: (block: Block) => void): string;
+    unsubscribeRuntimeVersion(id: string): void;
+    setHead(head: Block): Promise<void>;
+}

package/dist/blockchain/head-state.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HeadState = exports.randomId = void 0;
+const randomId = () => Math.random().toString(36).substring(2);
+exports.randomId = randomId;
+class HeadState {
+    #headListeners = {};
+    #storageListeners = {};
+    #oldValues = {};
+    #head;
+    constructor(head) {
+        this.#head = head;
+    }
+    subscribeHead(cb) {
+        const id = (0, exports.randomId)();
+        this.#headListeners[id] = cb;
+        return id;
+    }
+    unsubscribeHead(id) {
+        delete this.#headListeners[id];
+    }
+    async subscribeStorage(keys, cb) {
+        const id = (0, exports.randomId)();
+        this.#storageListeners[id] = [keys, cb];
+        for (const key of keys) {
+            this.#oldValues[key] = await this.#head.get(key);
+        }
+        return id;
+    }
+    unsubscribeStorage(id) {
+        delete this.#storageListeners[id];
+    }
+    subscrubeRuntimeVersion(cb) {
+        // TODO: actually subscribe
+        void cb;
+        return (0, exports.randomId)();
+    }
+    unsubscribeRuntimeVersion(id) {
+        // TODO: actually unsubscribe
+        void id;
+    }
+    async setHead(head) {
+        this.#head = head;
+        for (const cb of Object.values(this.#headListeners)) {
+            cb(head);
+        }
+        const diff = await this.#head.storageDiff();
+        for (const [keys, cb] of Object.values(this.#storageListeners)) {
+            const changed = keys.filter((key) => diff[key]).map((key) => [key, diff[key]]);
+            if (changed.length > 0) {
+                cb(head, changed);
+            }
+        }
+        Object.assign(this.#oldValues, diff);
+    }
+}
+exports.HeadState = HeadState;

package/dist/blockchain/index.d.ts

@@ -0,0 +1,35 @@
+import { DataSource } from 'typeorm';
+import { Header } from '@polkadot/types/interfaces';
+import { Api } from '../api';
+import { Block } from './block';
+import { BuildBlockMode } from './txpool';
+import { HeadState } from './head-state';
+import { InherentProvider } from './inherents';
+import { TaskManager } from '../task';
+export interface Options {
+    api: Api;
+    tasks: TaskManager;
+    buildBlockMode?: BuildBlockMode;
+    inherentProvider: InherentProvider;
+    db?: DataSource;
+    header: {
+        number: number;
+        hash: string;
+    };
+}
+export declare class Blockchain {
+    #private;
+    readonly api: Api;
+    readonly tasks: TaskManager;
+    readonly db: DataSource | undefined;
+    readonly headState: HeadState;
+    constructor({ api, tasks, buildBlockMode, inherentProvider, db, header }: Options);
+    get head(): Block;
+    getBlockAt(number?: number): Promise<Block | undefined>;
+    getBlock(hash?: string): Promise<Block | undefined>;
+    newTempBlock(parent: Block, header: Header): Block;
+    unregisterBlock(block: Block): void;
+    setHead(block: Block): void;
+    submitExtrinsic(extrinsic: string): Promise<string>;
+    newBlock(): Promise<Block>;
+}

package/dist/blockchain/index.js

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Blockchain = void 0;
+const util_crypto_1 = require("@polkadot/util-crypto");
+const util_1 = require("@polkadot/util");
+const block_1 = require("./block");
+const txpool_1 = require("./txpool");
+const head_state_1 = require("./head-state");
+const shared_1 = require("../rpc/shared");
+const logger_1 = require("../logger");
+const logger = logger_1.defaultLogger.child({ name: 'blockchain' });
+class Blockchain {
+    api;
+    tasks;
+    db;
+    #txpool;
+    #head;
+    #blocksByNumber = [];
+    #blocksByHash = {};
+    headState;
+    constructor({ api, tasks, buildBlockMode, inherentProvider, db, header }) {
+        this.api = api;
+        this.tasks = tasks;
+        this.db = db;
+        this.#head = new block_1.Block(this, header.number, header.hash);
+        this.#registerBlock(this.#head);
+        this.#txpool = new txpool_1.TxPool(this, inherentProvider, buildBlockMode);
+        this.headState = new head_state_1.HeadState(this.#head);
+    }
+    #registerBlock(block) {
+        this.#blocksByNumber[block.number] = block;
+        this.#blocksByHash[block.hash] = block;
+    }
+    get head() {
+        return this.#head;
+    }
+    async getBlockAt(number) {
+        if (number === undefined) {
+            return this.head;
+        }
+        if (number > this.#head.number) {
+            return undefined;
+        }
+        if (!this.#blocksByNumber[number]) {
+            const hash = await this.api.getBlockHash(number);
+            const block = new block_1.Block(this, number, hash);
+            this.#registerBlock(block);
+        }
+        return this.#blocksByNumber[number];
+    }
+    async getBlock(hash) {
+        await this.api.isReady;
+        if (hash == null) {
+            hash = this.head.hash;
+        }
+        if (!this.#blocksByHash[hash]) {
+            try {
+                const registry = await this.head.registry;
+                const header = registry.createType('Header', await this.api.getHeader(hash));
+                const block = new block_1.Block(this, header.number.toNumber(), hash);
+                this.#registerBlock(block);
+            }
+            catch (e) {
+                logger.debug(`getBlock(${hash}) failed: ${e}`);
+                return undefined;
+            }
+        }
+        return this.#blocksByHash[hash];
+    }
+    newTempBlock(parent, header) {
+        const number = parent.number + 1;
+        const hash = '0x' +
+            Math.round(Math.random() * 100000000)
+                .toString(16)
+                .padEnd(64, '0');
+        const block = new block_1.Block(this, number, hash, parent, { header, extrinsics: [], storage: parent.storage });
+        this.#blocksByHash[hash] = block;
+        return block;
+    }
+    unregisterBlock(block) {
+        if (block.hash === this.head.hash) {
+            throw new Error('Cannot unregister head block');
+        }
+        if (this.#blocksByNumber[block.number]?.hash === block.hash) {
+            delete this.#blocksByNumber[block.number];
+        }
+        delete this.#blocksByHash[block.hash];
+    }
+    setHead(block) {
+        logger.debug({
+            number: block.number,
+            hash: block.hash,
+        }, 'setHead');
+        this.#head = block;
+        this.#registerBlock(block);
+        this.headState.setHead(block);
+    }
+    async submitExtrinsic(extrinsic) {
+        const source = '0x02'; // External
+        const args = (0, util_1.u8aToHex)((0, util_1.u8aConcat)(source, extrinsic, this.head.hash));
+        const res = await this.head.call('TaggedTransactionQueue_validate_transaction', args);
+        const registry = await this.head.registry;
+        const validity = registry.createType('TransactionValidity', res.result);
+        if (validity.isOk) {
+            this.#txpool.submitExtrinsic(extrinsic);
+            return (0, util_crypto_1.blake2AsHex)(extrinsic, 256);
+        }
+        throw new shared_1.ResponseError(1, `Extrinsic is invalid: ${validity.asErr.toString()}`);
+    }
+    async newBlock() {
+        await this.#txpool.buildBlock();
+        return this.#head;
+    }
+}
+exports.Blockchain = Blockchain;

package/dist/blockchain/inherents.d.ts

@@ -0,0 +1,26 @@
+import { Block } from './block';
+import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types';
+import { TaskManager } from '../task';
+export interface CreateInherents {
+    createInherents(meta: DecoratedMeta, timestamp: number, parent: Block): Promise<string[]>;
+}
+export interface InherentProvider extends CreateInherents {
+    getTimestamp(): number;
+}
+export declare class SetTimestamp implements InherentProvider {
+    #private;
+    constructor(getTimestamp?: () => number);
+    createInherents(meta: DecoratedMeta, timestamp: number, _parent: Block): Promise<string[]>;
+    getTimestamp(): number;
+}
+export declare class InherentProviders implements InherentProvider {
+    #private;
+    constructor(base: InherentProvider, providers: CreateInherents[]);
+    createInherents(meta: DecoratedMeta, timestamp: number, parent: Block): Promise<string[]>;
+    getTimestamp(): number;
+}
+export declare class SetValidationData implements CreateInherents {
+    #private;
+    constructor(tasks: TaskManager, expectedIndex: number);
+    createInherents(meta: DecoratedMeta, _timestamp: number, parent: Block): Promise<string[]>;
+}