dequanto 0.0.1 → 0.0.7

package/src/hardhat/HardhatProvider.ts

@@ -0,0 +1,207 @@
+import memd from 'memd';
+import type Ethers from 'ethers'
+import type { ContractBase } from '@dequanto/contracts/ContractBase';
+import type { Constructor } from 'atma-utils/mixin';
+import { ChainAccount } from "@dequanto/models/TAccount";
+import { HardhatWeb3Client } from '@dequanto/clients/HardhatWeb3Client';
+import { class_Uri } from 'atma-utils';
+import { File, env, Directory } from 'atma-io';
+import { Web3Client } from '@dequanto/clients/Web3Client';
+import { ethers } from 'ethers';
+import { $logger } from '@dequanto/utils/$logger';
+import { $number } from '@dequanto/utils/$number';
+
+
+export class HardhatProvider {
+
+    /* lazy load */
+    hh = require('hardhat');
+
+    constructor() {
+        if (this.hh.ethers == null) {
+            throw new Error(`hardhat-ethers plugin should be installed and included in hardhat.config.js`)
+        }
+        if (this.hh.web3 == null) {
+            throw new Error(`hardhat-web3 plugin should be installed and included in hardhat.config.js`)
+        }
+    }
+
+
+    @memd.deco.memoize()
+    client(network: 'hardhat' | 'localhost' = 'hardhat') {
+        if (network == 'localhost') {
+            return new HardhatWeb3Client({
+                endpoints: [
+                    { url: 'http://127.0.0.1:8545' },
+                    // Use `manual`, will be used for subscriptions only, otherwise BatchRequests will fail, as not implemented yet
+                    // https://github.com/NomicFoundation/hardhat/issues/1324
+                    { url: 'ws://127.0.0.1:8545', manual: true },
+                ]
+            });
+        }
+        const web3 = this.hh.web3;
+        const client = new HardhatWeb3Client({ web3 });
+        return client;
+    }
+
+    @memd.deco.memoize()
+    deployer(index: number = 0): ChainAccount {
+        const ethers: typeof Ethers = this.hh.ethers;
+        const accounts: any = this.hh.config.networks.hardhat.accounts;
+        const wallet = ethers.Wallet.fromMnemonic(accounts.mnemonic, accounts.path + `/${index}`);
+        return {
+            key: wallet.privateKey,
+            address: wallet.address,
+        };
+    }
+
+    @memd.deco.memoize()
+    async deployClass<T extends ContractBase>(Ctor: Constructor<T>, options?: {
+        deployer?: Ethers.Signer | ChainAccount
+        arguments?: any[]
+        client?: Web3Client
+    }): Promise<T> {
+        let ethers: typeof Ethers = this.hh.ethers;
+
+        let client = options?.client ?? this.client();
+        let signer = options?.deployer ?? this.deployer();
+        let params = options?.arguments ?? [];
+
+        let Factory: Ethers.ContractFactory = await this.getFactory([Ctor.name], client, signer);
+
+        const contract = await Factory.deploy(...params);
+        const receipt = await contract.deployed();
+
+        $logger.log(`Contract ${Ctor.name} deployed to ${contract.address}`);
+
+        return new Ctor(contract.address, client);
+    }
+
+    async deploySol (solContractPath: string, options?: {
+        client?: Web3Client
+        arguments?: any[],
+        deployer?:  Ethers.Signer | ChainAccount,
+        paths?: {
+            root?: string
+            artifacts?: string
+        }
+    }): Promise<{ contract: Ethers.Contract, abi, bytecode }> {
+
+        const client = options?.client ?? this.client();
+        const args = options?.arguments ?? [];
+        const signer = options?.deployer ?? this.deployer();
+
+        const dir = solContractPath.replace(/[^\/]+$/, '');
+        const filename = /(?<filename>[^\/]+)\.\w+$/.exec(solContractPath)?.groups.filename;
+
+        let root = options?.paths?.root;
+        let artifacts = options?.paths?.artifacts;
+
+        if (filename == null) {
+            throw new Error(`Filename not extracted from ${solContractPath}`);
+        }
+
+        let hhOptions = {
+            sources: dir,
+            root,
+            artifacts,
+            tsgen: false,
+        };
+
+        await this.hh.run('compile', hhOptions);
+
+        if (root == null) {
+            root = 'file://' + process.cwd();
+        }
+        if (artifacts == null && root != null) {
+            artifacts = class_Uri.combine(root, 'artifacts/');
+        }
+
+        if (root != null && solContractPath.toLowerCase().includes(root.toLowerCase())) {
+            let i = solContractPath.toLowerCase().indexOf(root.toLowerCase());
+            solContractPath = solContractPath.substring(i + root.length);
+        }
+
+        let outputDir = class_Uri.combine(artifacts, solContractPath);
+        let output = class_Uri.combine(outputDir, `${filename}.json`);
+        console.log(`Compiled OK Output ${output} outputDir ${outputDir}`);
+
+        if (await File.existsAsync(output) === false) {
+            let path = `${outputDir}/`;
+            if (await Directory.existsAsync(path) === false) {
+                throw new Error(`No JSONs found in ${outputDir}, nor ${output}`);
+            }
+            let files = await Directory.readFilesAsync(path);
+            let jsons = files.filter(x => /(?<!dbg)\.json$/.test(x.uri.file));
+            if (jsons.length === 0) {
+                throw new Error(`No JSONs output found in ${outputDir}`);
+            }
+            if (jsons.length === 1) {
+                output = jsons[0].uri.toString();
+            } else {
+                let jsonFile = jsons.find(file => {
+                    return filename.includes(file.uri.filename);
+                });
+                if (jsonFile == null) {
+                    throw new Error(`Compiled JSON data not found for ${filename} in ${outputDir}`);
+                }
+                output = jsonFile.uri.toString();
+            }
+        }
+
+        let { abi, bytecode } = await File.readAsync<{ abi, bytecode }> (output);
+        let Factory: Ethers.ContractFactory = await this.getFactory([abi, bytecode], client, signer);
+
+        const contract = await Factory.deploy(...args);
+        const receipt = await contract.deployed();
+        return {
+            contract,
+            abi,
+            bytecode
+        };
+    }
+
+    async deployCode (solidityCode: string, options?: Parameters<HardhatProvider['deploySol']>[1]) {
+        let className = /contract\s+(?<name>[\w]+)/.exec(solidityCode).groups.name;
+        let rnd = $number.randomInt(0, 10**10);
+        let tmp = env.getTmpPath(`hardhat/contracts/${className}_${rnd}.sol`);
+        let root = tmp.replace(/contracts\/[^/]+$/, '');
+
+        options.paths = {
+            root
+        };
+        await File.writeAsync(tmp, solidityCode);
+        try {
+            return await this.deploySol(tmp, options);
+        } finally {
+            await Directory.removeAsync(root);
+        }
+
+    }
+
+    private getEthersProvider (client: Web3Client) {
+        if (client.options.web3 != null) {
+            let ethers: typeof Ethers & { provider /* hardhat */ } = this.hh.ethers;
+            return ethers.provider;
+        }
+
+        let url = client.options?.endpoints[0].url;
+        if (url.startsWith('ws')) {
+            return new ethers.providers.WebSocketProvider(url);
+        }
+        return new ethers.providers.JsonRpcProvider(url);
+    }
+
+    private async getFactory (factoryArgs: [string] | [any, string], client: Web3Client, signer: Ethers.Signer | ChainAccount): Promise<Ethers.ContractFactory> {
+        let ethers: typeof Ethers = this.hh.ethers;
+        let Factory: Ethers.ContractFactory = await (ethers as any).getContractFactory(...factoryArgs);
+
+        let provider = this.getEthersProvider(client);
+        let $signer = 'key' in signer
+            ? new ethers.Wallet(signer.key, provider)
+            : signer as Ethers.Signer;
+
+        Factory = Factory.connect($signer);
+        return Factory;
+    }
+}

package/src/indexer/BlocksTxIndexer.ts

@@ -1,5 +1,5 @@
-import { BlockTransactionString } from 'web3-eth';
-import { Transaction } from 'web3-core';
+import type { BlockTransactionString } from 'web3-eth';
+import type { Transaction, TransactionReceipt } from 'web3-core';
 import { BlocksWalker } from './handlers/BlocksWalker';
 import { Web3Client } from '@dequanto/clients/Web3Client';
 import { TPlatform } from '@dequanto/models/TPlatform';
@@ -10,10 +10,23 @@ import { $logger } from '@dequanto/utils/$logger';
 export interface IBlocksTxIndexerOptions {
     /** Name of the indexer */
     name?: string
+
+    /** Save indexer progress (visited blocks) to a file */
     persistance?: boolean
+
+    /** Load transactions from the block and provide them to the visitor method */
     loadTransactions?: boolean
+
+    /** Load receipts from the block and provide them to the visitor method */
+    loadReceipts?: boolean
+
+    client?: Web3Client
 }
-export type TBlockListener = (client: Web3Client, block: BlockTransactionString, txs?: Transaction[]) => Promise<void>
+export type TBlockListener = (
+    client: Web3Client,
+    block: BlockTransactionString,
+    data?: { txs?: Transaction[], receipts?: TransactionReceipt[] }
+) => Promise<void>
 
 export class BlocksTxIndexer {
     private client: Web3Client;
@@ -22,16 +35,18 @@ export class BlocksTxIndexer {
 
     constructor (public platform: TPlatform, public opts?: IBlocksTxIndexerOptions) {
 
-        this.client = Web3ClientFactory.get(platform, {
+        this.client = this.opts.client ?? Web3ClientFactory.get(platform, {
             ws: true
         });
         this.walker = new BlocksWalker({
             name: `${opts?.name ?? 'indexer'}_${this.platform}`,
             client: this.client,
             loadTransactions: opts?.loadTransactions ?? true,
+            loadReceipts: opts?.loadReceipts ?? false,
             persistance: opts?.persistance ?? true,
-            visitor: async (block, txs) => {
-                return this.indexTransactions(block, txs)
+
+            visitor: async (block, data) => {
+                return this.indexTransactions(block, data)
             }
         });
     }
@@ -49,17 +64,28 @@ export class BlocksTxIndexer {
         await this.walker.start(from, to);
 
         if (to == null) {
-            this.client.subscribe('newBlockHeaders', (error, blockHeader) => {
-                this.walker.processUntil(blockHeader.number);
+            await this.client.subscribe('newBlockHeaders', (error, blockHeader) => {
+                if (error) {
+                    $logger.error(`Subscription to "newBlockHeaders" failed with`, error);
+                    return;
+                }
+                if (blockHeader.transactions?.length === 0) {
+                    // hardhat emits empty blocks
+                    return;
+                }
+                this.walker.processUntil(blockHeader.number + 1);
             });
+            // Reload the blocknumber, to ensure we didn't missed the block between walker starting and subscription
+            let newTo = await this.client.getBlockNumber();
+            this.walker.processUntil(newTo + 1);
         }
     }
 
-    private async indexTransactions (block: BlockTransactionString, txs: Transaction[]) {
+    private async indexTransactions (block: BlockTransactionString, data: { txs?: Transaction[], receipts?: TransactionReceipt[] }) {
         for (let i = 0; i < this.listeners.length; i++) {
             let indexer = this.listeners[i];
             try {
-                await indexer(this.client, block, txs);
+                await indexer(this.client, block, data);
             } catch (error) {
                 $logger.log(error);
             }

package/src/indexer/handlers/BlocksWalker.ts

@@ -1,18 +1,20 @@
 import di from 'a-di';
 import alot from 'alot';
 import memd from 'memd';
-import { $array } from '@dequanto/utils/$array';
-import { FileSafe } from 'atma-io';
-import { PackedRanges } from '../../structs/PackedRanges';
 import { Everlog } from 'everlog';
-import { Web3Client } from '@dequanto/clients/Web3Client';
+import { FileSafe } from 'atma-io';
+import { class_Dfr } from 'atma-utils';
 import { BlockTransactionString } from 'web3-eth';
-import { Transaction } from 'web3-core';
-import { BlockDateResolver } from '@dequanto/blocks/BlockDateResolver';
+import { Transaction, TransactionReceipt } from 'web3-core';
+import { PackedRanges } from '../../structs/PackedRanges';
+
+import { $array } from '@dequanto/utils/$array';
 import { $date } from '@dequanto/utils/$date';
-import { class_Dfr } from 'atma-utils';
-import { $is } from '@dequanto/utils/$is';
 import { $logger } from '@dequanto/utils/$logger';
+import { $block } from '@dequanto/utils/$block';
+import { $require } from '@dequanto/utils/$require';
+
+import { Web3Client } from '@dequanto/clients/Web3Client';
 
 interface IBlockIndexer {
     name: string
@@ -21,22 +23,28 @@ interface IBlockIndexer {
         from: number
         to?: number
     }
-    visitor: (block: BlockTransactionString, txs: Transaction[]) => Promise<void>
+    visitor: (block: BlockTransactionString, data: { txs?: Transaction[], receipts?: TransactionReceipt[] }) => Promise<void>
     client: Web3Client
-    loadTransactions?: boolean
 
+    loadTransactions?: boolean
+    loadReceipts?: boolean
 }
 
 export class BlocksWalker {
-    private onEndPromise: class_Dfr;
+    public onEndPromise = new class_Dfr;
+
     private client = this.params.client;
     private visitor = this.params.visitor;
     private status = {
         blockLoadTime: 0
     };
 
-    private cachedState = new FileSafe(`./db/block-indexers/${this.params.name}.json`);
-    private fs = Everlog.createChannel(`indexer-${this.params.name}`, {
+    private cachedState = new FileSafe(`./0xweb/block-indexers/${this.params.name}.json`, {
+        cached: false,
+        processSafe: true,
+        threadSafe: true
+    });
+    private everlog = Everlog.createChannel(`indexer-${this.params.name}`, {
         fields: [
             { name: 'Date', type: 'date' },
             { name: 'Total', type: 'number' },
@@ -62,11 +70,7 @@ export class BlocksWalker {
             await this.restore();
         }
 
-        let onlyIncoming = from == null && to == null;
-        if (onlyIncoming) {
-            return;
-        }
-        if (from == null) {
+        if (to != null && from == null) {
             throw new Error(`FromBlock should be set while ToBlock(${to}) is present`);
         }
 
@@ -77,24 +81,26 @@ export class BlocksWalker {
             await this.getBlockNumber(to)
         );
 
-        $is.Number(this.ranges.from, 'From should be a number');
-        $is.Number(this.ranges.to, 'To should be a number');
+        $require.Number(this.ranges.from, 'From should be a number');
+        $require.Number(this.ranges.to, 'To should be a number');
 
         this.walker.process();
-        $logger.log(`BlocksWalker starting. Processing: ${this.ranges.from}-${to}. Completed: ${this.ranges.totalAdded()}; ToDo: ${this.ranges.totalLeft()}`);
+        $logger.log(`BlocksWalker starting. Processing: ${this.ranges.from}-${ to ?? 'latest' }. Completed: ${this.ranges.totalAdded()}; ToDo: ${this.ranges.totalLeft()}`);
     }
 
 
     /**
      *  Can be called each time we get a new block from blockchain,
      *  the walker will process its current blocks and up until the specified number
+     *  @param nr Number is not included: [from, end)
      */
     async processUntil (nr: number) {
         if (this.isRestored === false) {
             await this.restore();
         }
         this.walker.process(nr);
-        return this.onEndPromise = new class_Dfr;
+        this.onEndPromise.defer();
+        return this.onEndPromise;
     }
 
 
@@ -108,7 +114,7 @@ export class BlocksWalker {
 
     private isRestored = false;
 
-    @memd.deco.memoize()
+    @memd.deco.memoize({ perInstance: true })
     private async restore () {
         try {
             if (this.params.persistance !== false) {
@@ -122,7 +128,7 @@ export class BlocksWalker {
 
         this.walker = new RangeWalker({
             range: this.ranges,
-            onVisit: (nr) => this.processBlock(nr),
+            onVisit: (nrs) => this.processBlocks(nrs),
             onResult: (error, nr) => {
                 this.save();
                 this.log()
@@ -162,33 +168,36 @@ export class BlocksWalker {
         if (mix == null) {
             return this.client.getBlockNumber();
         }
-        if (typeof mix === 'number') {
-            return mix;
-        }
-        if (mix instanceof Date) {
-            let dateResolver = di.resolve(BlockDateResolver, this.client)
-            return await dateResolver.getBlockNumberFor(mix);
-        }
-        throw new Error(`Invalid getBlockNumber param: ${mix}`);
+        return $block.ensureNumber(mix, this.client);
     }
 
-    private async processBlock (nr: number) {
+    private async processBlocks (nrs: number[]) {
         // reading block and transactions
         let start = Date.now();
-        let block = await this.client.getBlock(nr);
-        let txs = null as Transaction[];
+        let blocks = await this.client.getBlocks(nrs);
+
+        let nr = await this.client.getBlockNumber();
 
-        if (this.params.loadTransactions !== false) {
+        let grouped = await alot(blocks).mapAsync(async block => {
             let hashes = block.transactions;
+            let txs = this.params.loadTransactions
+                ? await this.client.getTransactions(hashes)
+                : null;
 
-            txs = await alot(hashes).mapAsync(async hash => {
-                return this.client.getTransaction(hash);
-            }).toArrayAsync({ threads: 8 });
-        }
+            let receipts = this.params.loadReceipts
+                ? await this.client.getTransactionReceipts(hashes)
+                : null;
 
+            return { block, txs, receipts };
+        }).toArrayAsync({ threads: 4 });
 
-        this.status.blockLoadTime = Date.now() - start;
-        await this.visitor(block, txs)
+        this.status.blockLoadTime = (Date.now() - start) / nrs.length;
+
+        await alot(grouped)
+            .forEachAsync(async ({block, txs, receipts}) => {
+                await this.visitor(block, { txs, receipts })
+            })
+            .toArrayAsync({ threads: 4 });
     }
 
     @memd.deco.throttle(1000 * 4)
@@ -212,7 +221,7 @@ export class BlocksWalker {
             this.walker.status.errors,
             error,
         ];
-        this.fs.writeRow(row);
+        this.everlog.writeRow(row);
         $logger.log(row.join());
     }
 }
@@ -223,6 +232,7 @@ class RangeWalker {
 
     opts = {
         threads: 1,
+        batch: 1,
         timeout: 20000,
         // log every N ms
         logTimeWindow: 5000,
@@ -256,17 +266,19 @@ class RangeWalker {
     public onCompleted: class_Dfr
     public readonly range: PackedRanges
 
-    private readonly onVisit: (nr: number) => Promise<void>
-    private readonly onResult: (error: Error, nr: number) => void
+    private readonly onVisit: (nrs: number[]) => Promise<void>
+    private readonly onResult: (error: Error, nrs: number[]) => void
     private readonly onComplete: () => void
 
     constructor(params: {
+        batch?: number
         range: PackedRanges
         onVisit: RangeWalker['onVisit']
         onResult?: RangeWalker['onResult']
         onComplete?: RangeWalker['onComplete']
     }) {
 
+        this.opts.batch = params.batch ?? this.opts.batch;
         this.range = params.range;
         this.onVisit = params.onVisit;
         this.onResult = params.onResult;
@@ -290,11 +302,12 @@ class RangeWalker {
         this.range.from = nr;
     }
     setTo (nr: number) {
+        nr += 1;
+
         let { from } = this.range;
         if (from != null && from > nr) {
             throw new Error(`To (${nr}) should be greater then From (${from})`);
         }
-
         this.range.to = nr;
     }
     process (toBlock?: number) {
@@ -341,34 +354,40 @@ class RangeWalker {
             return;
         }
 
-        let next = this.range.next();
-        if (next == null) {
+        let arr = [] as number[];
+        while (arr.length < this.opts.batch) {
+            let next = this.range.next();
+            if (next == null) {
+                break;
+            }
+            arr.push(next);
+        }
+
+        if (arr.length === 0) {
             this.onComplete?.();
             return null;
         }
-        let workerData = {
-            nr: next,
-            startedAt: Date.now()
-        };
+        let workersData = arr.map(nr => ({ nr, startedAt: Date.now() }));
         let error: Error = null;
         try {
-            this.busy.push(workerData);
-            await this.onVisit(next);
+            this.busy.push(...workersData);
+            await this.onVisit(arr);
         } catch (err) {
             $logger.log('BlocksWalker.tick error', err);
             error = err;
         }
 
-        this.onResult?.(error, workerData.nr);
-        this.onTickComplete(workerData, error)
+        this.onResult?.(error, arr);
+        this.onTickComplete(workersData, error)
     }
 
-    private onTickComplete (workerData: { nr: number, startedAt: number }, error?: Error) {
+    private onTickComplete (workersData: { nr: number, startedAt: number }[], error?: Error) {
 
+        let lastWorkerData = workersData[workersData.length - 1];
         if (error != null) {
             this.lastError = {
-                ...workerData,
-                duration: Date.now() - workerData.startedAt,
+                ...lastWorkerData,
+                duration: Date.now() - lastWorkerData.startedAt,
                 error
             };
 
@@ -377,14 +396,14 @@ class RangeWalker {
             $logger.log(error);
         }
 
-        let time = Date.now() - workerData.startedAt;
         let prevAvgTime = this.status.avgTime;
         let prevTotal = this.status.processed;
 
+        let time = (Date.now() - lastWorkerData.startedAt) / workersData.length;
         this.status.avgTime = Math.round((prevAvgTime * prevTotal + time) / (prevTotal + 1));
-        this.status.processed += 1;
+        this.status.processed += workersData.length;
 
-        $array.remove(this.busy, workerData);
+        workersData.forEach(x => $array.remove(this.busy, x));
         this.tick();
     }
 }

package/src/indexer/handlers/TxQueueLoader.ts

@@ -3,35 +3,35 @@ import { $number } from '@dequanto/utils/$number';
 import { Transaction } from 'web3-core';
 
 export class TxQueueLoader {
-    opts = {
+    private opts = {
         threads: 4,
         timeout: 20000,
         // log every N ms
         logTimeWindow: 5000,
     };
 
-    status = {
+    private status = {
         errors: 0,
         processed: 0,
         avgLoadTime: 0,
     }
 
-    busy = [] as TxLoader[]
+    private busy = [] as TxLoader[]
 
-    errors = [] as {
+    private errors = [] as {
         date: Date
         hash: string
         error: Error
     }[]
 
-    lastError: {
+    private lastError: {
         date: Date
         hash: string
         error: Error
     }
 
-    queue: string[] = []
-    seen: string[] = []
+    private queue: string[] = []
+    private seen: string[] = []
 
     push (hash: string) {
         if (this.seen.includes(hash) === false) {

package/src/json/JsonObjectStore.ts

@@ -29,7 +29,7 @@ export class JsonObjectStore<T> {
     }
 
 
-    async get(opts?: { cloned?: boolean }): Promise<T> {
+    async get(opts?: { cloned?: boolean }): Promise<this['options']['default']> {
         if (this.fs.errored != null) {
             throw this.fs.errored;
         }

package/src/loggers/LoggerService.ts

@@ -1,3 +1,4 @@
+import { $date } from '@dequanto/utils/$date';
 import { Everlog } from 'everlog';
 import { ILogger } from './ILogger';
 
@@ -18,22 +19,25 @@ export class LoggerService implements ILogger {
     }
 
     warn (message: string) {
-        this.write([ new Date(), 'Warn', message ]);
+        this.write([ 'Warn', message ]);
     }
     error (message: string) {
-        this.write([ new Date(), 'Error', message ]);
+        this.write([ 'Error', message ]);
     }
     info (message: string) {
-        this.write([ new Date(), 'Info', message ]);
+        this.write([ 'Info', message ]);
     }
 
     write (row: string)
     write (row: any[])
     write (row: any[] | string) {
         let arr = typeof row === 'string' ? [ row ] : row;
-        arr.unshift(new Date());
+        let date = new Date();
+        arr.unshift(date);
         this.channel.writeRow(arr);
-        console.log(Array.isArray(row) ? row.join(' ') : row);
+
+        arr[0] = $date.format(date, 'HH:mm:ss');
+        console.log(arr.join(' '));
         return this;
     }
 }