@acala-network/chopsticks 0.4.2 → 0.5.0

package/lib/blockchain/block-builder.d.ts

@@ -2,7 +2,7 @@ import { Header, TransactionValidityError } from '@polkadot/types/interfaces';
 import { Block, TaskCallResponse } from './block';
 import { HexString } from '@polkadot/util/types';
 export declare const newHeader: (head: Block) => Promise<Header>;
-export declare const buildBlock: (head: Block, inherents: HexString[], extrinsics: HexString[], onApplyExtrinsicError: (extrinsic: HexString, error: TransactionValidityError) => void) => Promise<[Block, HexString[]]>;
+export declare const buildBlock: (head: Block, inherents: HexString[], extrinsics: HexString[], ump: Record<number, HexString[]>, onApplyExtrinsicError: (extrinsic: HexString, error: TransactionValidityError) => void) => Promise<[Block, HexString[]]>;
 export declare const dryRunExtrinsic: (head: Block, inherents: HexString[], extrinsic: HexString | {
     call: HexString;
     address: string;

package/lib/blockchain/block-builder.js

@@ -73,6 +73,7 @@ const newHeader = async (head) => {
             ...consensus.rest,
         ];
         if (meta.query.randomness) {
+            // TODO: shouldn't modify existing head
             // reset notFirstBlock so randomness will skip validation
             head.pushStorageLayer().set((0, utils_1.compactHex)(meta.query.randomness.notFirstBlock()), "Deleted" /* StorageValueKind.Deleted */);
         }
@@ -101,11 +102,14 @@ const initNewBlock = async (head, header, inherents) => {
         newBlock.pushStorageLayer().setAll(storageDiff);
         logger.trace((0, logger_1.truncate)(storageDiff), 'Initialize block');
     }
+    const layers = [];
     // apply inherents
     for (const extrinsic of inherents) {
         try {
             const { storageDiff } = await newBlock.call('BlockBuilder_apply_extrinsic', [extrinsic]);
-            newBlock.pushStorageLayer().setAll(storageDiff);
+            const layer = newBlock.pushStorageLayer();
+            layer.setAll(storageDiff);
+            layers.push(layer);
             logger.trace((0, logger_1.truncate)(storageDiff), 'Applied inherent');
         }
         catch (e) {
@@ -113,12 +117,15 @@ const initNewBlock = async (head, header, inherents) => {
             throw new Error('Failed to apply inherents');
         }
     }
-    return newBlock;
+    return {
+        block: newBlock,
+        layers: layers,
+    };
 };
-const buildBlock = async (head, inherents, extrinsics, onApplyExtrinsicError) => {
+const buildBlock = async (head, inherents, extrinsics, ump, onApplyExtrinsicError) => {
     const registry = await head.registry;
     const header = await (0, exports.newHeader)(head);
-    const newBlock = await initNewBlock(head, header, inherents);
+    const { block: newBlock } = await initNewBlock(head, header, inherents);
     logger.info({
         number: newBlock.number,
         extrinsicsCount: extrinsics.length,
@@ -126,18 +133,37 @@ const buildBlock = async (head, inherents, extrinsics, onApplyExtrinsicError) =>
     }, `Try building block #${newBlock.number.toLocaleString()}`);
     const pendingExtrinsics = [];
     const includedExtrinsic = [];
+    // apply ump via storage override hack
+    if (Object.keys(ump).length > 0) {
+        const meta = await head.meta;
+        const layer = head.pushStorageLayer();
+        for (const [paraId, upwardMessages] of Object.entries(ump)) {
+            const queueSize = meta.registry.createType('(u32, u32)', [
+                upwardMessages.length,
+                upwardMessages.map((x) => x.length).reduce((s, i) => s + i, 0),
+            ]);
+            const messages = meta.registry.createType('Vec<Bytes>', upwardMessages);
+            // TODO: make sure we append instead of replace
+            layer.setAll([
+                [(0, utils_1.compactHex)(meta.query.ump.relayDispatchQueues(paraId)), messages.toHex()],
+                [(0, utils_1.compactHex)(meta.query.ump.relayDispatchQueueSize(paraId)), queueSize.toHex()],
+            ]);
+        }
+        logger.trace({
+            number: newBlock.number,
+            tempHash: newBlock.hash,
+            ump,
+        }, 'Upward messages');
+        const needsDispatch = meta.registry.createType('Vec<u32>', Object.keys(ump));
+        layer.set((0, utils_1.compactHex)(meta.query.ump.needsDispatch()), needsDispatch.toHex());
+    }
     // apply extrinsics
     for (const extrinsic of extrinsics) {
         try {
             const { result, storageDiff } = await newBlock.call('BlockBuilder_apply_extrinsic', [extrinsic]);
             const outcome = registry.createType('ApplyExtrinsicResult', result);
             if (outcome.isErr) {
-                if (outcome.asErr.isInvalid && outcome.asErr.asInvalid.isFuture) {
-                    pendingExtrinsics.push(extrinsic);
-                }
-                else {
-                    onApplyExtrinsicError(extrinsic, outcome.asErr);
-                }
+                onApplyExtrinsicError(extrinsic, outcome.asErr);
                 continue;
             }
             newBlock.pushStorageLayer().setAll(storageDiff);
@@ -179,7 +205,7 @@ exports.buildBlock = buildBlock;
 const dryRunExtrinsic = async (head, inherents, extrinsic) => {
     const registry = await head.registry;
     const header = await (0, exports.newHeader)(head);
-    const newBlock = await initNewBlock(head, header, inherents);
+    const { block: newBlock } = await initNewBlock(head, header, inherents);
     if (typeof extrinsic !== 'string') {
         if (!head.chain.mockSignatureHost) {
             throw new Error('Cannot fake signature because mock signature host is not enabled. Start chain with `mockSignatureHost: true`');
@@ -208,7 +234,11 @@ const dryRunExtrinsic = async (head, inherents, extrinsic) => {
 exports.dryRunExtrinsic = dryRunExtrinsic;
 const dryRunInherents = async (head, inherents) => {
     const header = await (0, exports.newHeader)(head);
-    const newBlock = await initNewBlock(head, header, inherents);
-    return Object.entries(await newBlock.storageDiff());
+    const { layers } = await initNewBlock(head, header, inherents);
+    const stoarge = {};
+    for (const layer of layers) {
+        await layer.mergeInto(stoarge);
+    }
+    return Object.entries(stoarge);
 };
 exports.dryRunInherents = dryRunInherents;

package/lib/blockchain/block.js

@@ -31,15 +31,17 @@ class Block {
         this.#extrinsics = block?.extrinsics;
         this.#baseStorage = block?.storage ?? new storage_layer_1.RemoteStorageLayer(chain.api, hash, chain.db);
         this.#storages = [];
-        const storageDiff = block?.storageDiff || {};
-        // if code doesn't change then reuse parent block's meta
-        if (!storageDiff[(0, util_2.stringToHex)(':code')]) {
-            this.#runtimeVersion = parentBlock?.runtimeVersion;
-            this.#metadata = parentBlock?.metadata;
-            this.#registry = parentBlock?.registry;
-            this.#meta = parentBlock?.meta;
+        const storageDiff = block?.storageDiff;
+        if (storageDiff) {
+            // if code doesn't change then reuse parent block's meta
+            if (!storageDiff?.[(0, util_2.stringToHex)(':code')]) {
+                this.#runtimeVersion = parentBlock?.runtimeVersion;
+                this.#metadata = parentBlock?.metadata;
+                this.#registry = parentBlock?.registry;
+                this.#meta = parentBlock?.meta;
+            }
+            this.pushStorageLayer().setAll(storageDiff);
         }
-        this.pushStorageLayer().setAll(storageDiff);
     }
     get chain() {
         return this.#chain;

package/lib/blockchain/head-state.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HeadState = exports.randomId = void 0;
-const rxjs_1 = require("rxjs");
 const util_1 = require("@polkadot/util");
 const logger_1 = require("../logger");
 const randomId = () => Math.random().toString(36).substring(2);
@@ -47,27 +46,23 @@ class HeadState {
     async setHead(head) {
         this.#head = head;
         for (const cb of Object.values(this.#headListeners)) {
-            rxjs_1.asapScheduler.schedule(() => {
-                try {
-                    cb(head);
-                }
-                catch (error) {
-                    logger.error(error, 'callback');
-                }
-            });
+            try {
+                await cb(head);
+            }
+            catch (error) {
+                logger.error(error, 'setHead head callback error');
+            }
         }
         const diff = await this.#head.storageDiff();
         for (const [keys, cb] of Object.values(this.#storageListeners)) {
             const changed = keys.filter((key) => diff[key]).map((key) => [key, diff[key]]);
             if (changed.length > 0) {
-                rxjs_1.asapScheduler.schedule(() => {
-                    try {
-                        cb(head, changed);
-                    }
-                    catch (error) {
-                        logger.error(error, 'callback');
-                    }
-                });
+                try {
+                    await cb(head, changed);
+                }
+                catch (error) {
+                    logger.error(error, 'setHead storaeg diff callback error');
+                }
             }
         }
         Object.assign(this.#oldValues, diff);

package/lib/blockchain/index.d.ts

@@ -4,7 +4,7 @@ import { HexString } from '@polkadot/util/types';
 import { RegisteredTypes } from '@polkadot/types/types';
 import { Api } from '../api';
 import { Block } from './block';
-import { BuildBlockMode, BuildBlockParams, DownwardMessage, HorizontalMessage, TxPool, UpcomingBlockParams } from './txpool';
+import { BuildBlockMode, BuildBlockParams, DownwardMessage, HorizontalMessage, TxPool } from './txpool';
 import { HeadState } from './head-state';
 import { InherentProvider } from './inherent';
 export interface Options {
@@ -37,8 +37,12 @@ export declare class Blockchain {
     unregisterBlock(block: Block): void;
     setHead(block: Block): Promise<void>;
     submitExtrinsic(extrinsic: HexString): Promise<HexString>;
-    newBlock(params?: BuildBlockParams): Promise<Block>;
-    upcomingBlock(params?: UpcomingBlockParams): Promise<Block>;
+    submitUpwardMessages(id: number, ump: HexString[]): void;
+    submitDownwardMessages(dmp: DownwardMessage[]): void;
+    submitHorizontalMessages(id: number, hrmp: HorizontalMessage[]): void;
+    newBlock(params?: Partial<BuildBlockParams>): Promise<Block>;
+    newBlockWithParams(params: BuildBlockParams): Promise<Block>;
+    upcomingBlocks(): Promise<number>;
     dryRunExtrinsic(extrinsic: HexString | {
         call: HexString;
         address: string;

package/lib/blockchain/index.js

@@ -118,12 +118,28 @@ class Blockchain {
         }
         throw validity.asErr;
     }
+    submitUpwardMessages(id, ump) {
+        this.#txpool.submitUpwardMessages(id, ump);
+        logger.debug({ id, ump }, 'submitUpwardMessages');
+    }
+    submitDownwardMessages(dmp) {
+        this.#txpool.submitDownwardMessages(dmp);
+        logger.debug({ dmp }, 'submitDownwardMessages');
+    }
+    submitHorizontalMessages(id, hrmp) {
+        this.#txpool.submitHorizontalMessages(id, hrmp);
+        logger.debug({ id, hrmp }, 'submitHorizontalMessages');
+    }
     async newBlock(params) {
         await this.#txpool.buildBlock(params);
         return this.#head;
     }
-    async upcomingBlock(params) {
-        return this.#txpool.upcomingBlock(params);
+    async newBlockWithParams(params) {
+        await this.#txpool.buildBlockWithParams(params);
+        return this.#head;
+    }
+    async upcomingBlocks() {
+        return this.#txpool.upcomingBlocks();
     }
     async dryRunExtrinsic(extrinsic, at) {
         await this.api.isReady;
@@ -132,7 +148,12 @@ class Blockchain {
             throw new Error(`Cannot find block ${at}`);
         }
         const registry = await head.registry;
-        const inherents = await this.#inherentProvider.createInherents(head);
+        const inherents = await this.#inherentProvider.createInherents(head, {
+            transactions: [],
+            downwardMessages: [],
+            upwardMessages: [],
+            horizontalMessages: {},
+        });
         const { result, storageDiff } = await (0, block_builder_1.dryRunExtrinsic)(head, inherents, extrinsic);
         const outcome = registry.createType('ApplyExtrinsicResult', result);
         return { outcome, storageDiff };
@@ -143,7 +164,12 @@ class Blockchain {
         if (!head) {
             throw new Error(`Cannot find block ${at}`);
         }
-        const inherents = await this.#inherentProvider.createInherents(head, { horizontalMessages: hrmp });
+        const inherents = await this.#inherentProvider.createInherents(head, {
+            transactions: [],
+            downwardMessages: [],
+            upwardMessages: [],
+            horizontalMessages: hrmp,
+        });
         return (0, block_builder_1.dryRunInherents)(head, inherents);
     }
     async dryRunDmp(dmp, at) {
@@ -152,7 +178,12 @@ class Blockchain {
         if (!head) {
             throw new Error(`Cannot find block ${at}`);
         }
-        const inherents = await this.#inherentProvider.createInherents(head, { downwardMessages: dmp });
+        const inherents = await this.#inherentProvider.createInherents(head, {
+            transactions: [],
+            downwardMessages: dmp,
+            upwardMessages: [],
+            horizontalMessages: {},
+        });
         return (0, block_builder_1.dryRunInherents)(head, inherents);
     }
     async dryRunUmp(ump, at) {
@@ -178,12 +209,22 @@ class Blockchain {
             stroageValues.push([(0, utils_1.compactHex)(meta.query.ump.relayDispatchQueueSize(paraId)), queueSize.toHex()]);
         }
         head.pushStorageLayer().setAll(stroageValues);
-        const inherents = await this.#inherentProvider.createInherents(head);
+        const inherents = await this.#inherentProvider.createInherents(head, {
+            transactions: [],
+            downwardMessages: [],
+            upwardMessages: [],
+            horizontalMessages: {},
+        });
         return (0, block_builder_1.dryRunInherents)(head, inherents);
     }
     async getInherents() {
         await this.api.isReady;
-        const inherents = await this.#inherentProvider.createInherents(this.head);
+        const inherents = await this.#inherentProvider.createInherents(this.head, {
+            transactions: [],
+            downwardMessages: [],
+            upwardMessages: [],
+            horizontalMessages: {},
+        });
         return inherents;
     }
 }

package/lib/blockchain/inherent/index.d.ts

@@ -6,7 +6,7 @@ export { ParaInherentEnter } from './para-enter';
 export { SetBabeRandomness } from './parachain/babe-randomness';
 export { SetNimbusAuthorInherent } from './parachain/nimbus-author-inherent';
 export interface CreateInherents {
-    createInherents(parent: Block, params?: BuildBlockParams['inherent']): Promise<HexString[]>;
+    createInherents(parent: Block, params: BuildBlockParams): Promise<HexString[]>;
 }
 export type InherentProvider = CreateInherents;
 export declare class SetTimestamp implements InherentProvider {
@@ -15,5 +15,5 @@ export declare class SetTimestamp implements InherentProvider {
 export declare class InherentProviders implements InherentProvider {
     #private;
     constructor(base: InherentProvider, providers: CreateInherents[]);
-    createInherents(parent: Block, params?: BuildBlockParams['inherent']): Promise<HexString[]>;
+    createInherents(parent: Block, params: BuildBlockParams): Promise<HexString[]>;
 }

package/lib/blockchain/inherent/para-enter.d.ts

@@ -3,5 +3,5 @@ import { Block } from '../block';
 import { BuildBlockParams } from '../txpool';
 import { CreateInherents } from '.';
 export declare class ParaInherentEnter implements CreateInherents {
-    createInherents(parent: Block, _params?: BuildBlockParams['inherent']): Promise<HexString[]>;
+    createInherents(parent: Block, _params: BuildBlockParams): Promise<HexString[]>;
 }

package/lib/blockchain/inherent/parachain/babe-randomness.d.ts

@@ -3,5 +3,5 @@ import { Block } from '../../block';
 import { BuildBlockParams } from '../../txpool';
 import { CreateInherents } from '..';
 export declare class SetBabeRandomness implements CreateInherents {
-    createInherents(parent: Block, _params?: BuildBlockParams['inherent']): Promise<HexString[]>;
+    createInherents(parent: Block, _params: BuildBlockParams): Promise<HexString[]>;
 }

package/lib/blockchain/inherent/parachain/nimbus-author-inherent.d.ts

@@ -3,5 +3,5 @@ import { Block } from '../../block';
 import { BuildBlockParams } from '../../txpool';
 import { CreateInherents } from '..';
 export declare class SetNimbusAuthorInherent implements CreateInherents {
-    createInherents(parent: Block, _params?: BuildBlockParams['inherent']): Promise<HexString[]>;
+    createInherents(parent: Block, _params: BuildBlockParams): Promise<HexString[]>;
 }

package/lib/blockchain/inherent/parachain/validation-data.d.ts

@@ -15,5 +15,5 @@ export type ValidationData = {
     };
 };
 export declare class SetValidationData implements CreateInherents {
-    createInherents(parent: Block, params?: BuildBlockParams['inherent']): Promise<HexString[]>;
+    createInherents(parent: Block, params: BuildBlockParams): Promise<HexString[]>;
 }

package/lib/blockchain/inherent/parachain/validation-data.js

@@ -75,7 +75,7 @@ class SetValidationData {
             // inject downward messages
             let dmqMqcHeadHash = decoded[dmqMqcHeadKey];
             if (dmqMqcHeadHash) {
-                for (const { msg, sentAt } of params?.downwardMessages || []) {
+                for (const { msg, sentAt } of params.downwardMessages) {
                     // calculate new hash
                     dmqMqcHeadHash = (0, util_crypto_1.blake2AsHex)((0, util_1.u8aConcat)(meta.registry.createType('Hash', dmqMqcHeadHash).toU8a(), meta.registry.createType('BlockNumber', sentAt).toU8a(), (0, util_crypto_1.blake2AsU8a)(meta.registry.createType('Bytes', msg).toU8a(), 256)), 256);
                     downwardMessages.push({
@@ -94,7 +94,7 @@ class SetValidationData {
             const hrmpMessages = {
                 // reset values, we just need the keys
                 ...lodash_1.default.mapValues(extrinsic.horizontalMessages, () => []),
-                ...(params?.horizontalMessages || {}),
+                ...params.horizontalMessages,
             };
             // inject horizontal messages
             for (const id of hrmpIngressChannels) {

package/lib/blockchain/storage-layer.js

@@ -128,7 +128,6 @@ class StorageLayer {
     }
     async getKeysPaged(prefix, pageSize, startKey) {
         if (!this.#deletedPrefix.some((prefix) => startKey.startsWith(prefix))) {
-            await this.fold();
             // TODO: maintain a list of fetched ranges to avoid fetching the same range multiple times
             const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? [];
             for (const key of remote) {

package/lib/blockchain/txpool.d.ts

@@ -1,7 +1,6 @@
 /// <reference types="node" />
 import { EventEmitter } from 'node:stream';
 import { HexString } from '@polkadot/util/types';
-import { Block } from './block';
 import { Blockchain } from '.';
 import { InherentProvider } from './inherent';
 export declare const APPLY_EXTRINSIC_ERROR = "TxPool::ApplyExtrinsicError";
@@ -19,14 +18,10 @@ export interface HorizontalMessage {
     data: HexString;
 }
 export interface BuildBlockParams {
-    inherent?: {
-        downwardMessages?: DownwardMessage[];
-        horizontalMessages?: Record<number, HorizontalMessage[]>;
-    };
-}
-export interface UpcomingBlockParams {
-    skipCount?: number;
-    timeout?: number;
+    downwardMessages: DownwardMessage[];
+    upwardMessages: Record<number, HexString[]>;
+    horizontalMessages: Record<number, HorizontalMessage[]>;
+    transactions: HexString[];
 }
 export declare class TxPool {
     #private;
@@ -34,6 +29,10 @@ export declare class TxPool {
     constructor(chain: Blockchain, inherentProvider: InherentProvider, mode?: BuildBlockMode);
     get pendingExtrinsics(): HexString[];
     submitExtrinsic(extrinsic: HexString): void;
-    buildBlock(params?: BuildBlockParams): Promise<void>;
-    upcomingBlock(params?: UpcomingBlockParams): Promise<Block>;
+    submitUpwardMessages(id: number, ump: HexString[]): void;
+    submitDownwardMessages(dmp: DownwardMessage[]): void;
+    submitHorizontalMessages(id: number, hrmp: HorizontalMessage[]): void;
+    buildBlockWithParams(params: BuildBlockParams): Promise<void>;
+    buildBlock(params?: Partial<BuildBlockParams>): Promise<void>;
+    upcomingBlocks(): Promise<number>;
 }

package/lib/blockchain/txpool.js

@@ -4,10 +4,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TxPool = exports.BuildBlockMode = exports.APPLY_EXTRINSIC_ERROR = void 0;
-const rxjs_1 = require("rxjs");
 const node_stream_1 = require("node:stream");
-const operators_1 = require("rxjs/operators");
 const lodash_1 = __importDefault(require("lodash"));
+const utils_1 = require("../utils");
 const block_builder_1 = require("./block-builder");
 exports.APPLY_EXTRINSIC_ERROR = 'TxPool::ApplyExtrinsicError';
 var BuildBlockMode;
@@ -19,14 +18,16 @@ var BuildBlockMode;
 class TxPool {
     #chain;
     #pool = [];
+    #ump = {};
+    #dmp = [];
+    #hrmp = {};
     #mode;
     #inherentProvider;
+    #pendingBlocks = [];
     event = new node_stream_1.EventEmitter();
-    #last;
-    #lastBuildBlockPromise = Promise.resolve();
+    #isBuilding = false;
     constructor(chain, inherentProvider, mode = BuildBlockMode.Batch) {
         this.#chain = chain;
-        this.#last = new rxjs_1.BehaviorSubject(chain.head);
         this.#mode = mode;
         this.#inherentProvider = inherentProvider;
     }
@@ -35,6 +36,27 @@ class TxPool {
     }
     submitExtrinsic(extrinsic) {
         this.#pool.push(extrinsic);
+        this.#maybeBuildBlock();
+    }
+    submitUpwardMessages(id, ump) {
+        if (!this.#ump[id]) {
+            this.#ump[id] = [];
+        }
+        this.#ump[id].push(...ump);
+        this.#maybeBuildBlock();
+    }
+    submitDownwardMessages(dmp) {
+        this.#dmp.push(...dmp);
+        this.#maybeBuildBlock();
+    }
+    submitHorizontalMessages(id, hrmp) {
+        if (!this.#hrmp[id]) {
+            this.#hrmp[id] = [];
+        }
+        this.#hrmp[id].push(...hrmp);
+        this.#maybeBuildBlock();
+    }
+    #maybeBuildBlock() {
         switch (this.#mode) {
             case BuildBlockMode.Batch:
                 this.#batchBuildBlock();
@@ -48,33 +70,73 @@ class TxPool {
         }
     }
     #batchBuildBlock = lodash_1.default.debounce(this.buildBlock, 100, { maxWait: 1000 });
+    async buildBlockWithParams(params) {
+        this.#pendingBlocks.push({
+            params,
+            deferred: (0, utils_1.defer)(),
+        });
+        this.#buildBlockIfNeeded();
+        await this.upcomingBlocks();
+    }
     async buildBlock(params) {
-        const last = this.#lastBuildBlockPromise;
-        this.#lastBuildBlockPromise = this.#buildBlock(last, params);
-        await this.#lastBuildBlockPromise;
-        this.#last.next(this.#chain.head);
+        const transactions = params?.transactions || this.#pool.splice(0);
+        const upwardMessages = params?.upwardMessages || { ...this.#ump };
+        const downwardMessages = params?.downwardMessages || this.#dmp.splice(0);
+        const horizontalMessages = params?.horizontalMessages || { ...this.#hrmp };
+        if (!params?.upwardMessages) {
+            for (const id of Object.keys(this.#ump)) {
+                delete this.#ump[id];
+            }
+        }
+        if (!params?.horizontalMessages) {
+            for (const id of Object.keys(this.#hrmp)) {
+                delete this.#hrmp[id];
+            }
+        }
+        await this.buildBlockWithParams({
+            transactions,
+            upwardMessages,
+            downwardMessages,
+            horizontalMessages,
+        });
     }
-    async upcomingBlock(params) {
-        const { skipCount, timeout: millisecs } = { skipCount: 0, ...(params || {}) };
-        if (skipCount < 0)
-            throw new Error('skipCount needs to be greater or equal to 0');
-        let stream$ = this.#last.pipe();
-        if (millisecs) {
-            stream$ = stream$.pipe((0, operators_1.timeout)(millisecs));
+    async upcomingBlocks() {
+        const count = this.#pendingBlocks.length;
+        if (count > 0) {
+            await this.#pendingBlocks[count - 1].deferred.promise;
         }
-        return (0, rxjs_1.firstValueFrom)(stream$.pipe((0, operators_1.skip)(1 + skipCount), (0, operators_1.take)(1)));
+        return count;
     }
-    async #buildBlock(wait, params) {
+    async #buildBlockIfNeeded() {
+        if (this.#isBuilding)
+            return;
+        if (this.#pendingBlocks.length === 0)
+            return;
+        this.#isBuilding = true;
+        try {
+            await this.#buildBlock();
+        }
+        finally {
+            this.#isBuilding = false;
+            this.#buildBlockIfNeeded();
+        }
+    }
+    async #buildBlock() {
         await this.#chain.api.isReady;
-        await wait.catch(() => { }); // ignore error
+        const pending = this.#pendingBlocks[0];
+        if (!pending) {
+            throw new Error('Unreachable');
+        }
+        const { params, deferred } = pending;
         const head = this.#chain.head;
-        const extrinsics = this.#pool.splice(0);
-        const inherents = await this.#inherentProvider.createInherents(head, params?.inherent);
-        const [newBlock, pendingExtrinsics] = await (0, block_builder_1.buildBlock)(head, inherents, extrinsics, (extrinsic, error) => {
+        const inherents = await this.#inherentProvider.createInherents(head, params);
+        const [newBlock, pendingExtrinsics] = await (0, block_builder_1.buildBlock)(head, inherents, params.transactions, params.upwardMessages, (extrinsic, error) => {
             this.event.emit(exports.APPLY_EXTRINSIC_ERROR, [extrinsic, error]);
         });
         this.#pool.push(...pendingExtrinsics);
         await this.#chain.setHead(newBlock);
+        this.#pendingBlocks.shift();
+        deferred.resolve();
     }
 }
 exports.TxPool = TxPool;

package/lib/cli.js

@@ -40,6 +40,7 @@ const defaultOptions = {
     },
     block: {
         desc: 'Block hash or block number. Default to latest block',
+        string: true,
     },
     'wasm-override': {
         desc: 'Path to wasm override',
@@ -146,7 +147,7 @@ const defaultOptions = {
     ...defaultOptions,
 }), async (argv) => {
     const context = await (0, _1.setup)(await processArgv(argv));
-    const { storage, decodedKey } = await (0, decoder_1.decodeKey)(context.chain.head, argv.key);
+    const { storage, decodedKey } = (0, decoder_1.decodeKey)(await context.chain.head.meta, context.chain.head, argv.key);
     if (storage && decodedKey) {
         console.log(`${storage.section}.${storage.method}`, decodedKey.args.map((x) => JSON.stringify(x.toHuman())).join(', '));
     }

package/lib/rpc/dev/index.js

@@ -8,13 +8,20 @@ const time_travel_1 = require("../../utils/time-travel");
 const logger = logger_1.defaultLogger.child({ name: 'rpc-dev' });
 const handlers = {
     dev_newBlock: async (context, [param]) => {
-        const { count, to, hrmp } = param || {};
+        const { count, to, hrmp, ump, dmp, transactions } = param || {};
         const now = context.chain.head.number;
         const diff = to ? to - now : count;
         const finalCount = diff > 0 ? diff : 1;
         let finalHash;
         for (let i = 0; i < finalCount; i++) {
-            const block = await context.chain.newBlock({ inherent: { horizontalMessages: hrmp || [] } }).catch((error) => {
+            const block = await context.chain
+                .newBlock({
+                transactions,
+                horizontalMessages: hrmp,
+                upwardMessages: ump,
+                downwardMessages: dmp,
+            })
+                .catch((error) => {
                 throw new shared_1.ResponseError(1, error.toString());
             });
             logger.debug({ hash: block.hash }, 'dev_newBlock');

package/lib/rpc/substrate/author.js

@@ -31,7 +31,6 @@ const handlers = {
             if (!extrisnics.includes(extrinsic))
                 return;
             logger.debug({ block: block.hash }, 'author_extrinsicUpdate');
-            // for now just assume tx is always included on next block
             callback({
                 InBlock: block.hash,
             });

package/lib/schema/index.d.ts

@@ -134,6 +134,7 @@ export declare const configSchema: z.ZodObject<{
     'registered-types': z.ZodOptional<z.ZodAny>;
 }, "strict", z.ZodTypeAny, {
     timestamp?: number | undefined;
+    block?: string | number | undefined;
     db?: string | undefined;
     genesis?: string | {
         name: string;
@@ -151,7 +152,6 @@ export declare const configSchema: z.ZodObject<{
     } | undefined;
     port?: number | undefined;
     endpoint?: string | undefined;
-    block?: string | number | undefined;
     'build-block-mode'?: BuildBlockMode | undefined;
     'import-storage'?: any;
     'mock-signature-host'?: boolean | undefined;
@@ -159,6 +159,7 @@ export declare const configSchema: z.ZodObject<{
     'registered-types'?: any;
 }, {
     timestamp?: number | undefined;
+    block?: string | number | undefined;
     db?: string | undefined;
     genesis?: string | {
         name: string;
@@ -176,7 +177,6 @@ export declare const configSchema: z.ZodObject<{
     } | undefined;
     port?: number | undefined;
     endpoint?: string | undefined;
-    block?: string | number | undefined;
     'build-block-mode'?: BuildBlockMode | undefined;
     'import-storage'?: any;
     'mock-signature-host'?: boolean | undefined;

package/lib/setup.js

@@ -30,11 +30,14 @@ const setup = async (argv) => {
     if (argv.block == null) {
         blockHash = await api.getBlockHash();
     }
+    else if (typeof argv.block === 'string' && argv.block.startsWith('0x')) {
+        blockHash = argv.block;
+    }
     else if (Number.isInteger(+argv.block)) {
         blockHash = await api.getBlockHash(Number(argv.block));
     }
     else {
-        blockHash = argv.block;
+        throw new Error(`Invalid block number or hash: ${argv.block}`);
     }
     logger_1.defaultLogger.debug({ ...argv, blockHash }, 'Args');
     let db;

package/lib/utils/decoder.d.ts

@@ -1,17 +1,18 @@
 import '@polkadot/types-codec';
 import { Block } from '../blockchain/block';
+import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types';
 import { HexString } from '@polkadot/util/types';
 import { StorageEntry } from '@polkadot/types/primitive/types';
 import { StorageKey } from '@polkadot/types';
-export declare const decodeKey: (block: Block, key: HexString) => Promise<{
+export declare const decodeKey: (meta: DecoratedMeta, block: Block, key: HexString) => {
     storage?: StorageEntry | undefined;
     decodedKey?: StorageKey<import("@polkadot/types-codec/types").AnyTuple> | undefined;
-}>;
-export declare const decodeKeyValue: (block: Block, key: HexString, value?: HexString | null) => Promise<{
+};
+export declare const decodeKeyValue: (meta: DecoratedMeta, block: Block, key: HexString, value?: HexString | null) => {
     [x: string]: `0x${string}` | null | undefined;
 } | {
     [x: string]: {
         [x: string]: import("@polkadot/types-codec/types").AnyJson;
     };
-}>;
+};
 export declare const decodeStorageDiff: (block: Block, diff: [HexString, HexString | null][]) => Promise<({} | undefined)[]>;

package/lib/utils/decoder.js

@@ -20,13 +20,12 @@ const getCache = (uid) => {
     }
     return _CACHE[uid];
 };
-const getStorageEntry = async (block, key) => {
+const getStorageEntry = (meta, block, key) => {
     const cache = getCache(block.chain.uid);
     for (const [prefix, storageEntry] of Object.entries(cache)) {
         if (key.startsWith(prefix))
             return storageEntry;
     }
-    const meta = await block.meta;
     for (const module of Object.values(meta.query)) {
         for (const storage of Object.values(module)) {
             const keyPrefix = (0, util_1.u8aToHex)(storage.keyPrefix());
@@ -36,11 +35,10 @@ const getStorageEntry = async (block, key) => {
             }
         }
     }
-    throw new Error(`Cannot find key ${key}`);
+    return undefined;
 };
-const decodeKey = async (block, key) => {
-    const meta = await block.meta;
-    const storage = await getStorageEntry(block, key).catch(() => undefined);
+const decodeKey = (meta, block, key) => {
+    const storage = getStorageEntry(meta, block, key);
     const decodedKey = meta.registry.createType('StorageKey', key);
     if (storage) {
         decodedKey.setMeta(storage.meta);
@@ -49,9 +47,8 @@ const decodeKey = async (block, key) => {
     return {};
 };
 exports.decodeKey = decodeKey;
-const decodeKeyValue = async (block, key, value) => {
-    const meta = await block.meta;
-    const { storage, decodedKey } = await (0, exports.decodeKey)(block, key);
+const decodeKeyValue = (meta, block, key, value) => {
+    const { storage, decodedKey } = (0, exports.decodeKey)(meta, block, key);
     if (!storage || !decodedKey) {
         return { [key]: value };
     }
@@ -96,9 +93,10 @@ exports.decodeKeyValue = decodeKeyValue;
 const decodeStorageDiff = async (block, diff) => {
     const oldState = {};
     const newState = {};
+    const meta = await block.meta;
     for (const [key, value] of diff) {
-        lodash_1.default.merge(oldState, await (0, exports.decodeKeyValue)(block, key, (await block.get(key))));
-        lodash_1.default.merge(newState, await (0, exports.decodeKeyValue)(block, key, value));
+        lodash_1.default.merge(oldState, (0, exports.decodeKeyValue)(meta, block, key, (await block.get(key))));
+        lodash_1.default.merge(newState, (0, exports.decodeKeyValue)(meta, block, key, value));
     }
     const oldStateWithoutEvents = lodash_1.default.cloneDeep(oldState);
     if (oldStateWithoutEvents['system']?.['events']) {

package/lib/utils/index.d.ts

@@ -8,3 +8,9 @@ export declare function fetchKeysToArray(getKeys: GetKeys): Promise<StorageKey<a
 export declare const compactHex: (value: Uint8Array) => HexString;
 export declare const getParaId: (chain: Blockchain) => Promise<import("@polkadot/types").u32>;
 export declare const isUrl: (url: string) => boolean;
+export type Deferred<T> = {
+    resolve: (value: T | PromiseLike<T>) => void;
+    reject: (reason?: any) => void;
+    promise: Promise<T>;
+};
+export declare function defer<T>(): Deferred<T>;

package/lib/utils/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isUrl = exports.getParaId = exports.compactHex = exports.fetchKeysToArray = exports.fetchKeys = void 0;
+exports.defer = exports.isUrl = exports.getParaId = exports.compactHex = exports.fetchKeysToArray = exports.fetchKeys = void 0;
 const util_1 = require("@polkadot/util");
 async function fetchKeys(getKeys, processKey) {
     const processKeys = async (keys) => {
@@ -48,3 +48,12 @@ const isUrl = (url) => {
     }
 };
 exports.isUrl = isUrl;
+function defer() {
+    const deferred = {};
+    deferred.promise = new Promise((resolve, reject) => {
+        deferred.resolve = resolve;
+        deferred.reject = reject;
+    });
+    return deferred;
+}
+exports.defer = defer;

package/lib/xcm/downward.js

@@ -23,7 +23,7 @@ const connectDownward = async (relaychain, parachain) => {
         if (downwardMessages.length === 0)
             return;
         _1.logger.debug({ downwardMessages }, 'downward_message');
-        await parachain.newBlock({ inherent: { downwardMessages } });
+        parachain.submitDownwardMessages(downwardMessages);
     });
 };
 exports.connectDownward = connectDownward;

package/lib/xcm/horizontal.js

@@ -18,12 +18,9 @@ const connectHorizontal = async (parachains) => {
                 .toJSON();
             _1.logger.info({ outboundHrmpMessage }, 'outboundHrmpMessage');
             for (const { recipient, data } of outboundHrmpMessage) {
-                const horizontalMessages = {
-                    [Number(id)]: [{ sentAt: head.number, data }],
-                };
                 const receiver = parachains[recipient];
                 if (receiver) {
-                    await receiver.newBlock({ inherent: { horizontalMessages } });
+                    receiver.submitHorizontalMessages(Number(id), [{ sentAt: head.number, data }]);
                 }
             }
         });

package/lib/xcm/upward.js

@@ -3,10 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.connectUpward = void 0;
 const util_1 = require("@polkadot/util");
 const utils_1 = require("../utils");
-const _1 = require(".");
 const connectUpward = async (parachain, relaychain) => {
     const meta = await parachain.head.meta;
-    const paraId = await (0, utils_1.getParaId)(parachain);
+    const paraId = (await (0, utils_1.getParaId)(parachain)).toNumber();
     const upwardMessagesKey = (0, utils_1.compactHex)(meta.query.parachainSystem.upwardMessages());
     await parachain.headState.subscribeStorage([upwardMessagesKey], async (_head, pairs) => {
         const value = pairs[0][1];
@@ -16,19 +15,7 @@ const connectUpward = async (parachain, relaychain) => {
         const upwardMessages = meta.registry.createType('Vec<Bytes>', (0, util_1.hexToU8a)(value));
         if (upwardMessages.length === 0)
             return;
-        const queueSize = meta.registry.createType('(u32, u32)', [
-            upwardMessages.length,
-            upwardMessages.map((x) => x.byteLength).reduce((s, i) => s + i, 0),
-        ]);
-        const needsDispatch = meta.registry.createType('Vec<u32>', [paraId]);
-        _1.logger.debug({ [paraId.toNumber()]: upwardMessages.toJSON(), queueSize: queueSize.toJSON() }, 'upward_message');
-        // TODO: make sure we append instead of replace
-        relaychain.head.pushStorageLayer().setAll([
-            [(0, utils_1.compactHex)(meta.query.ump.needsDispatch()), needsDispatch.toHex()],
-            [(0, utils_1.compactHex)(meta.query.ump.relayDispatchQueues(paraId)), upwardMessages.toHex()],
-            [(0, utils_1.compactHex)(meta.query.ump.relayDispatchQueueSize(paraId)), queueSize.toHex()],
-        ]);
-        await relaychain.newBlock();
+        relaychain.submitUpwardMessages(paraId, upwardMessages.map((x) => x.toHex()));
     });
 };
 exports.connectUpward = connectUpward;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@acala-network/chopsticks",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
   "author": "Bryan Chen <xlchen1291@gmail.com>",
@@ -20,7 +20,7 @@
     "dev:moonbeam": "ts-node-dev --transpile-only --inspect --notify=false src/cli.ts -- dev --config=../../configs/moonbeam.yml"
   },
   "dependencies": {
-    "@acala-network/chopsticks-executor": "0.4.2",
+    "@acala-network/chopsticks-executor": "0.5.0",
     "@polkadot/api": "^9.14.2",
     "@polkadot/rpc-provider": "^9.14.2",
     "@polkadot/types": "^9.14.2",