@acala-network/chopsticks-core 1.2.0 → 1.2.2

package/dist/cjs/rpc/rpc-spec/index.js

@@ -22,6 +22,7 @@ _export(exports, {
         return _default;
     }
 });
+const _archive_v1 = /*#__PURE__*/ _interop_require_wildcard(require("./archive_v1.js"));
 const _chainHead_v1 = /*#__PURE__*/ _interop_require_wildcard(require("./chainHead_v1.js"));
 const _chainSpec_v1 = /*#__PURE__*/ _interop_require_wildcard(require("./chainSpec_v1.js"));
 const _transaction_v1 = /*#__PURE__*/ _interop_require_wildcard(require("./transaction_v1.js"));
@@ -67,6 +68,7 @@ function _interop_require_wildcard(obj, nodeInterop) {
     return newObj;
 }
 const handlers = {
+    ..._archive_v1,
     ..._chainHead_v1,
     ..._transaction_v1,
     ..._chainSpec_v1

package/dist/cjs/rpc/rpc-spec/storage-common.d.ts

@@ -0,0 +1,16 @@
+import type { HexString } from '@polkadot/util/types';
+import type { Block } from '../../blockchain/block.js';
+export declare function getDescendantValues(block: Block, params: DescendantValuesParams): Promise<{
+    items: Array<{
+        key: string;
+        value?: HexString;
+    }>;
+    next: DescendantValuesParams | null;
+}>;
+export declare const PAGE_SIZE = 1000;
+export type DescendantValuesParams = {
+    prefix: string;
+    startKey: string;
+    isDescendantHashes?: boolean;
+};
+export declare function afterResponse(fn: () => void): Promise<void>;

package/dist/cjs/rpc/rpc-spec/storage-common.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get PAGE_SIZE () {
+        return PAGE_SIZE;
+    },
+    get afterResponse () {
+        return afterResponse;
+    },
+    get getDescendantValues () {
+        return getDescendantValues;
+    }
+});
+async function getDescendantValues(block, params) {
+    const keys = await block.getKeysPaged({
+        ...params,
+        pageSize: PAGE_SIZE
+    });
+    const items = (await block.getMany(keys)).map((value, idx)=>({
+            key: keys[idx],
+            value
+        }));
+    if (keys.length < PAGE_SIZE) {
+        return {
+            items,
+            next: null
+        };
+    }
+    return {
+        items,
+        next: {
+            ...params,
+            startKey: keys[PAGE_SIZE - 1]
+        }
+    };
+}
+const PAGE_SIZE = 1000;
+async function afterResponse(fn) {
+    await new Promise((resolve)=>setTimeout(resolve, 0));
+    fn();
+}

package/dist/esm/blockchain/block.d.ts

@@ -48,6 +48,10 @@ export declare class Block {
      * Get the block storage by key.
      */
     get(key: string): Promise<HexString | undefined>;
+    /**
+     * Get the block storage by key.
+     */
+    getMany(keys: string[]): Promise<Array<HexString | undefined>>;
     read<T extends string>(type: T, query: StorageEntry, ...args: any[]): Promise<import("@polkadot/types/types").DetectCodec<import("@polkadot/types-codec/types").Codec, T> | undefined>;
     /**
      * Get paged storage keys.

package/dist/esm/blockchain/block.js

@@ -117,6 +117,19 @@ import { RemoteStorageLayer, StorageLayer, StorageValueKind } from './storage-la
                 return val;
         }
     }
+    /**
+   * Get the block storage by key.
+   */ async getMany(keys) {
+        const vals = await this.storage.getMany(keys, true);
+        return vals.map((val)=>{
+            switch(val){
+                case StorageValueKind.Deleted:
+                    return undefined;
+                default:
+                    return val;
+            }
+        });
+    }
     async read(type, query, ...args) {
         const key = compactHex(query(...args));
         const value = await this.get(key);

package/dist/esm/blockchain/inherent/parachain/validation-data.js

@@ -2,9 +2,13 @@ import { GenericExtrinsic } from '@polkadot/types';
 import { hexToU8a, u8aConcat, u8aToHex } from '@polkadot/util';
 import { blake2AsHex, blake2AsU8a } from '@polkadot/util-crypto';
 import _ from 'lodash';
+import { defaultLogger } from '../../../logger.js';
 import { compactHex, getCurrentSlot, getParaId } from '../../../utils/index.js';
 import { dmqMqcHead, hrmpChannels, hrmpEgressChannelIndex, hrmpIngressChannelIndex, paraHead, upgradeGoAheadSignal, WELL_KNOWN_KEYS } from '../../../utils/proof.js';
 import { createProof, decodeProof } from '../../../wasm-executor/index.js';
+const logger = defaultLogger.child({
+    name: 'parachain-validation-data'
+});
 const MOCK_VALIDATION_DATA = {
     validationData: {
         relayParentNumber: 1000,
@@ -31,7 +35,7 @@ const MOCK_VALIDATION_DATA = {
     horizontalMessages: [],
     downwardMessages: []
 };
-const getValidationData = async (parent)=>{
+const getValidationData = async (parent, fallback = true)=>{
     const meta = await parent.meta;
     if (parent.number === 0) {
         const { trieRootHash, nodes } = await createProof(MOCK_VALIDATION_DATA.relayChainState.trieNodes, []);
@@ -46,15 +50,39 @@ const getValidationData = async (parent)=>{
             }
         };
     }
-    const extrinsics = await parent.extrinsics;
-    const validationDataExtrinsic = extrinsics.find((extrinsic)=>{
-        const firstArg = meta.registry.createType('GenericExtrinsic', extrinsic)?.args?.[0];
-        return firstArg && 'validationData' in firstArg;
-    });
-    if (!validationDataExtrinsic) {
-        throw new Error('Missing validation data from block');
+    try {
+        const extrinsics = await parent.extrinsics;
+        const validationDataExtrinsic = extrinsics.find((extrinsic)=>{
+            const firstArg = meta.registry.createType('GenericExtrinsic', extrinsic)?.args?.[0];
+            return firstArg && 'validationData' in firstArg;
+        });
+        if (!validationDataExtrinsic) {
+            throw new Error('Missing validation data from block');
+        }
+        return meta.registry.createType('GenericExtrinsic', validationDataExtrinsic).args[0].toJSON();
+    } catch (e) {
+        logger.warn('Failed to get validation data from block %d %s', parent.number, e);
+        if (fallback) {
+            // this could fail due to wasm override that breaks the validation data format
+            // so we will try parent's parent
+            const grandParent = await parent.parentBlock;
+            if (grandParent) {
+                const data = await getValidationData(grandParent, false);
+                return {
+                    ...data,
+                    validationData: {
+                        ...data.validationData,
+                        relayParentNumber: data.validationData.relayParentNumber + 2
+                    }
+                };
+            } else {
+                throw e;
+            }
+        } else {
+            // fallback failed, throw error
+            throw e;
+        }
     }
-    return meta.registry.createType('GenericExtrinsic', validationDataExtrinsic).args[0].toJSON();
 };
 export class SetValidationData {
     async createInherents(newBlock, params) {
@@ -82,7 +110,12 @@ export class SetValidationData {
             if (key === WELL_KNOWN_KEYS.CURRENT_SLOT) {
                 // increment current slot
                 const relayCurrentSlot = decoded[key] ? meta.registry.createType('Slot', hexToU8a(decoded[key])).toNumber() : await getCurrentSlot(parent) * relaySlotIncrease;
-                const newSlot = meta.registry.createType('Slot', relayCurrentSlot + relaySlotIncrease);
+                const newSlot = meta.registry.createType('Slot', relayCurrentSlot + relaySlotIncrease + 1) // +1 to be safe
+                ;
+                logger.debug({
+                    relayCurrentSlot,
+                    newSlot: newSlot.toNumber()
+                }, 'Updating relay current slot');
                 newEntries.push([
                     key,
                     u8aToHex(newSlot.toU8a())

package/dist/esm/blockchain/storage-layer.d.ts

@@ -7,10 +7,18 @@ export declare enum StorageValueKind {
 }
 export type StorageValue = string | StorageValueKind | undefined;
 export interface StorageLayerProvider {
+    /**
+     * Returns true if key is deleted
+     */
+    deleted(key: string): boolean;
     /**
      * Get the value of a storage key.
      */
     get(key: string, cache: boolean): Promise<StorageValue>;
+    /**
+     * Get the value of many storage keys.
+     */
+    getMany(keys: string[], _cache: boolean): Promise<StorageValue[]>;
     /**
      * Get paged storage keys.
      */
@@ -23,14 +31,18 @@ export interface StorageLayerProvider {
 export declare class RemoteStorageLayer implements StorageLayerProvider {
     #private;
     constructor(api: Api, at: HexString, db: Database | undefined);
+    deleted(_key: string): boolean;
     get(key: string, _cache: boolean): Promise<StorageValue>;
+    getMany(keys: string[], _cache: boolean): Promise<StorageValue[]>;
     findNextKey(prefix: string, startKey: string, _knownBest?: string): Promise<string | undefined>;
     getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise<string[]>;
 }
 export declare class StorageLayer implements StorageLayerProvider {
     #private;
     constructor(parent?: StorageLayerProvider);
+    deleted(key: string): boolean;
     get(key: string, cache: boolean): Promise<StorageValue | undefined>;
+    getMany(keys: string[], cache: boolean): Promise<StorageValue[]>;
     set(key: string, value: StorageValue): void;
     setAll(values: Record<string, StorageValue | null> | [string, StorageValue | null][]): void;
     findNextKey(prefix: string, startKey: string, knownBest?: string): Promise<string | undefined>;

package/dist/esm/blockchain/storage-layer.js

@@ -22,6 +22,9 @@ export class RemoteStorageLayer {
         this.#at = at;
         this.#db = db;
     }
+    deleted(_key) {
+        return false;
+    }
     async get(key, _cache) {
         if (this.#db) {
             const res = await this.#db.queryStorage(this.#at, key);
@@ -37,6 +40,48 @@ export class RemoteStorageLayer {
         this.#db?.saveStorage(this.#at, key, data);
         return data ?? undefined;
     }
+    async getMany(keys, _cache) {
+        const result = [];
+        let pending = keys.map((key, idx)=>({
+                key,
+                idx
+            }));
+        if (this.#db) {
+            const results = await Promise.all(pending.map(({ key })=>this.#db.queryStorage(this.#at, key)));
+            const oldPending = pending;
+            pending = [];
+            results.forEach((res, idx)=>{
+                if (res) {
+                    result[idx] = res.value ?? undefined;
+                } else {
+                    pending.push({
+                        key: oldPending[idx].key,
+                        idx
+                    });
+                }
+            });
+        }
+        if (pending.length) {
+            logger.trace({
+                at: this.#at,
+                keys
+            }, 'RemoteStorageLayer getMany');
+            const data = await this.#api.getStorageBatch('0x', pending.map(({ key })=>key), this.#at);
+            data.forEach(([, res], idx)=>{
+                result[pending[idx].idx] = res ?? undefined;
+            });
+            if (this.#db?.saveStorageBatch) {
+                this.#db?.saveStorageBatch(data.map(([key, value])=>({
+                        key,
+                        value,
+                        blockHash: this.#at
+                    })));
+            } else if (this.#db) {
+                data.forEach(([key, value])=>this.#db?.saveStorage(this.#at, key, value));
+            }
+        }
+        return result;
+    }
     async findNextKey(prefix, startKey, _knownBest) {
         const keys = await this.getKeysPaged(prefix, 1, startKey);
         return keys[0];
@@ -134,6 +179,18 @@ export class StorageLayer {
             this.#keys.splice(idx, 1);
         }
     }
+    deleted(key) {
+        if (this.#store.has(key)) {
+            return this.#store.get(key) === "Deleted";
+        }
+        if (this.#deletedPrefix.some((dp)=>key.startsWith(dp))) {
+            return true;
+        }
+        if (this.#parent) {
+            return this.#parent.deleted(key);
+        }
+        return false;
+    }
     async get(key, cache) {
         if (this.#store.has(key)) {
             return this.#store.get(key);
@@ -150,6 +207,33 @@ export class StorageLayer {
         }
         return undefined;
     }
+    async getMany(keys, cache) {
+        const result = [];
+        const pending = [];
+        const preloadedPromises = keys.map(async (key, idx)=>{
+            if (this.#store.has(key)) {
+                result[idx] = await this.#store.get(key);
+            } else if (this.#deletedPrefix.some((dp)=>key.startsWith(dp))) {
+                result[idx] = "Deleted";
+            } else {
+                pending.push({
+                    key,
+                    idx
+                });
+            }
+        });
+        if (pending.length && this.#parent) {
+            const vals = await this.#parent.getMany(pending.map((p)=>p.key), false);
+            vals.forEach((val, idx)=>{
+                if (cache) {
+                    this.#store.set(pending[idx].key, val);
+                }
+                result[pending[idx].idx] = val;
+            });
+        }
+        await Promise.all(preloadedPromises);
+        return result;
+    }
     set(key, value) {
         switch(value){
             case "Deleted":
@@ -212,7 +296,7 @@ export class StorageLayer {
             const next = await this.findNextKey(prefix, startKey, undefined);
             if (!next) break;
             startKey = next;
-            if (await this.get(next, false) === "Deleted") continue;
+            if (this.deleted(next)) continue;
             keys.push(next);
         }
         return keys;

package/dist/esm/rpc/rpc-spec/archive_v1.d.ts

@@ -0,0 +1,74 @@
+import type { HexString } from '@polkadot/util/types';
+import { type Handler } from '../shared.js';
+import type { StorageItemRequest } from './chainHead_v1.js';
+/**
+ * Retrieve the body of a specific block
+ *
+ * @param context
+ * @param params - [`hash`]
+ *
+ * @return An array of the SCALE-encoded transactions of a block, or `null` if the block is not found.
+ */
+export declare const archive_v1_body: Handler<[HexString], HexString[] | null>;
+export type CallResult = {
+    success: true;
+    value: HexString;
+} | {
+    success: false;
+    error: any;
+};
+export declare const archive_v1_call: Handler<[HexString, string, HexString], CallResult | null>;
+/**
+ * Retrieve the height of the finalized block.
+ *
+ * @param context
+ *
+ * @return The `number` of the height of the head (a.k.a. finalized) block.
+ */
+export declare const archive_v1_finalizedHeight: Handler<undefined, number>;
+/**
+ * Retrieve the genesis hash
+ *
+ * @param context
+ *
+ * @return An {@link HexString} with the hash of the genesis block.
+ */
+export declare const archive_v1_genesisHash: Handler<undefined, HexString>;
+/**
+ * Retrieve the hash of a specific height
+ *
+ * @param context
+ * @param params - [`height`]
+ *
+ * @return An array of {@link HexString} with the hashes of the blocks associated to the
+ * given height.
+ */
+export declare const archive_v1_hashByHeight: Handler<[number], HexString[]>;
+/**
+ * Retrieve the header for a specific block
+ *
+ * @param context
+ * @param params - [`hash`]
+ *
+ * @return SCALE-encoded header, or `null` if the block is not found.
+ */
+export declare const archive_v1_header: Handler<[HexString], HexString | null>;
+/**
+ * Query the storage for a given block
+ *
+ * @param context
+ * @param params - [`hash`, `items`, `childTrie`]
+ *
+ * @return the operationId to capture the notifications where to receive the result
+ *
+ * The query type `closestDescendantMerkleValue` is not up to spec.
+ * According to the spec, the result should be the Merkle value of the key or
+ * the closest descendant of the key.
+ * As chopsticks doesn't have direct access to the Merkle tree, it will return
+ * a string that will change every time that one of the descendant changes, but
+ * it won't be the actual Merkle value.
+ * This should be enough for applications that don't rely on the actual Merkle
+ * value, but just use it to detect for storage changes.
+ */
+export declare const archive_v1_storage: Handler<[HexString, StorageItemRequest[], HexString | null], string>;
+export declare const archive_v1_stopStorage: Handler<[string], null>;

package/dist/esm/rpc/rpc-spec/archive_v1.js

@@ -0,0 +1,215 @@
+import { blake2AsHex } from '@polkadot/util-crypto';
+import { randomId } from '../../blockchain/head-state.js';
+import { ResponseError } from '../shared.js';
+import { archive_unstable_body, archive_unstable_call } from '../substrate/archive.js';
+import { afterResponse, getDescendantValues } from './storage-common.js';
+/**
+ * Retrieve the body of a specific block
+ *
+ * @param context
+ * @param params - [`hash`]
+ *
+ * @return An array of the SCALE-encoded transactions of a block, or `null` if the block is not found.
+ */ export const archive_v1_body = async (...args)=>archive_unstable_body(...args).catch(()=>null);
+/**
+ * Perform a runtime call for a block
+ *
+ * @param context
+ * @param params - [`hash`, `function`, `callParameters`]
+ *
+ * @return A {@link CallResult} with the result of the runtime call, or `null` if the block
+ * is not found.
+ */ function isBlockNotFound(error) {
+    return error instanceof ResponseError && error.code === 1;
+}
+export const archive_v1_call = async (...args)=>archive_unstable_call.call(undefined, ...args).then(({ value })=>({
+            success: true,
+            value
+        }), (error)=>isBlockNotFound(error) ? null : {
+            success: false,
+            error
+        });
+/**
+ * Retrieve the height of the finalized block.
+ *
+ * @param context
+ *
+ * @return The `number` of the height of the head (a.k.a. finalized) block.
+ */ export const archive_v1_finalizedHeight = (context)=>{
+    return Promise.resolve(context.chain.head.number);
+};
+/**
+ * Retrieve the genesis hash
+ *
+ * @param context
+ *
+ * @return An {@link HexString} with the hash of the genesis block.
+ */ export const archive_v1_genesisHash = async (context)=>{
+    const genesisBlock = await context.chain.getBlockAt(0);
+    return genesisBlock.hash;
+};
+/**
+ * Retrieve the hash of a specific height
+ *
+ * @param context
+ * @param params - [`height`]
+ *
+ * @return An array of {@link HexString} with the hashes of the blocks associated to the
+ * given height.
+ */ export const archive_v1_hashByHeight = async (context, [height])=>{
+    const block = await context.chain.getBlockAt(height);
+    return block ? [
+        block.hash
+    ] : [];
+};
+/**
+ * Retrieve the header for a specific block
+ *
+ * @param context
+ * @param params - [`hash`]
+ *
+ * @return SCALE-encoded header, or `null` if the block is not found.
+ */ export const archive_v1_header = async (context, [hash])=>{
+    const block = await context.chain.getBlock(hash);
+    return block ? (await block.header).toHex() : null;
+};
+/**
+ * Contains the storage operations.
+ */ const storageOperations = new Map();
+/**
+ * Query the storage for a given block
+ *
+ * @param context
+ * @param params - [`hash`, `items`, `childTrie`]
+ *
+ * @return the operationId to capture the notifications where to receive the result
+ *
+ * The query type `closestDescendantMerkleValue` is not up to spec.
+ * According to the spec, the result should be the Merkle value of the key or
+ * the closest descendant of the key.
+ * As chopsticks doesn't have direct access to the Merkle tree, it will return
+ * a string that will change every time that one of the descendant changes, but
+ * it won't be the actual Merkle value.
+ * This should be enough for applications that don't rely on the actual Merkle
+ * value, but just use it to detect for storage changes.
+ */ export const archive_v1_storage = async (context, [hash, items, _childTrie], { subscribe })=>{
+    const operationId = randomId();
+    const callback = subscribe('chainHead_v1_storageEvent', operationId, ()=>storageOperations.delete(operationId));
+    storageOperations.set(operationId, {
+        callback,
+        hash,
+        params: [],
+        storageDiffs: new Map()
+    });
+    afterResponse(async ()=>{
+        const block = await context.chain.getBlock(hash);
+        if (!block) {
+            storageOperations.get(operationId)?.callback({
+                event: 'storageError',
+                operationId,
+                error: 'Block not found'
+            });
+            return;
+        }
+        const handleStorageItemRequest = async (sir)=>{
+            switch(sir.type){
+                case 'value':
+                    {
+                        const value = await block.get(sir.key);
+                        if (value) {
+                            storageOperations.get(operationId)?.callback({
+                                event: 'storage',
+                                key: sir.key,
+                                value
+                            });
+                        }
+                        return null;
+                    }
+                case 'hash':
+                    {
+                        const value = await block.get(sir.key);
+                        if (value) {
+                            storageOperations.get(operationId)?.callback({
+                                event: 'storage',
+                                key: sir.key,
+                                hash
+                            });
+                        }
+                        return null;
+                    }
+                case 'descendantsValues':
+                    {
+                        let items;
+                        let next = {
+                            prefix: sir.key,
+                            startKey: '0x'
+                        };
+                        do {
+                            ;
+                            ({ items, next } = await getDescendantValues(block, next));
+                            for (const { key, value } of items){
+                                storageOperations.get(operationId)?.callback({
+                                    event: 'storage',
+                                    key,
+                                    value
+                                });
+                            }
+                        }while (next !== null)
+                        return null;
+                    }
+                case 'descendantsHashes':
+                    {
+                        let items;
+                        let next = {
+                            prefix: sir.key,
+                            startKey: '0x'
+                        };
+                        do {
+                            ;
+                            ({ items, next } = await getDescendantValues(block, next));
+                            for (const { key, value } of items){
+                                if (value === undefined) {
+                                    continue;
+                                }
+                                storageOperations.get(operationId)?.callback({
+                                    event: 'storage',
+                                    key,
+                                    hash: blake2AsHex(value)
+                                });
+                            }
+                        }while (next !== null)
+                        return null;
+                    }
+                case 'closestDescendantMerkleValue':
+                    {
+                        const subscription = storageOperations.get(operationId);
+                        if (!subscription) return null;
+                        if (!subscription.storageDiffs.has(sir.key)) {
+                            // Set up a diff watch for this key
+                            subscription.storageDiffs.set(sir.key, 0);
+                        }
+                        subscription.callback({
+                            event: 'storage',
+                            operationId,
+                            items: [
+                                {
+                                    key: sir.key,
+                                    closestDescendantMerkleValue: String(subscription.storageDiffs.get(sir.key))
+                                }
+                            ]
+                        });
+                        return null;
+                    }
+            }
+        };
+        await Promise.all(items.map(handleStorageItemRequest));
+        storageOperations.get(operationId)?.callback({
+            event: 'storageDone'
+        });
+    });
+    return operationId;
+};
+export const archive_v1_stopStorage = async (_, [operationId], { unsubscribe })=>{
+    unsubscribe(operationId);
+    return null;
+};