@lido-nestjs/registry 1.1.1 → 1.2.0

package/dist/main/abstract-registry.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 import { LoggerService } from '@nestjs/common';
 import { Registry } from '@lido-nestjs/contracts';
-import { EntityManager } from '@mikro-orm/core';
+import { EntityManager } from '@mikro-orm/knex';
 import EventEmitter from 'events';
 import { CronJob } from 'cron';
 import { RegistryMetaFetchService } from '../fetch/meta.fetch';
@@ -47,6 +47,10 @@ export declare abstract class AbstractRegistryService {
     getMetaDataFromStorage(): Promise<RegistryMeta | null>;
     /** returns the latest operators data from the db */
     getOperatorsFromStorage(): Promise<RegistryOperator[]>;
+    /** returns all the keys from storage */
+    getOperatorsKeysFromStorage(): Promise<RegistryKey[]>;
+    /** saves all the data to the db */
+    save(updatedKeys: RegistryKey[], currentOperators: RegistryOperator[], currMeta: RegistryMeta): Promise<void>;
     /** clears the db */
     clear(): Promise<void>;
 }

package/dist/main/abstract-registry.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
 var tslib = require('tslib');
 var common = require('@nestjs/common');
 var contracts = require('@lido-nestjs/contracts');
-var core = require('@mikro-orm/core');
+var knex = require('@mikro-orm/knex');
 var logger = require('@lido-nestjs/logger');
 var decorators = require('@lido-nestjs/decorators');
 var EventEmitter = require('events');
@@ -22,6 +22,7 @@ var operator_entity = require('../storage/operator.entity.js');
 var meta_utils = require('../utils/meta.utils.js');
 var operator_utils = require('../utils/operator.utils.js');
 var constants = require('./constants.js');
+var utils = require('@lido-nestjs/utils');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
@@ -40,7 +41,7 @@ exports.AbstractRegistryService = class AbstractRegistryService {
         this.entityManager = entityManager;
         this.options = options;
         this.eventEmitter = new EventEmitter__default["default"]();
-        this.cronJob = new cron.CronJob((options === null || options === void 0 ? void 0 : options.subscribeInterval) || '*/10 * * * * *', this.cronHandler);
+        this.cronJob = new cron.CronJob((options === null || options === void 0 ? void 0 : options.subscribeInterval) || '*/10 * * * * *', () => this.cronHandler());
     }
     async cronHandler() {
         try {
@@ -85,7 +86,10 @@ exports.AbstractRegistryService = class AbstractRegistryService {
         }
         if (isSameContractState) {
             (_c = (_b = this.logger).debug) === null || _c === void 0 ? void 0 : _c.call(_b, 'Same state, no data update required', { currMeta });
-            await this.metaStorage.save(currMeta);
+            await this.entityManager.transactional(async (entityManager) => {
+                await entityManager.nativeDelete(meta_entity.RegistryMeta, {});
+                await entityManager.persist(new meta_entity.RegistryMeta(currMeta));
+            });
             (_e = (_d = this.logger).debug) === null || _e === void 0 ? void 0 : _e.call(_d, 'Updated metadata in the DB', { currMeta });
             return;
         }
@@ -100,19 +104,7 @@ exports.AbstractRegistryService = class AbstractRegistryService {
         this.logger.log('Fetched updated keys', {
             updatedKeys: updatedKeys.length,
         });
-        // save all data in a transaction
-        await this.entityManager.transactional(async (entityManager) => {
-            updatedKeys.forEach(async (operatorKey) => {
-                const instance = new key_entity.RegistryKey(operatorKey);
-                entityManager.persist(instance);
-            });
-            currentOperators.forEach(async (operator) => {
-                const instance = new operator_entity.RegistryOperator(operator);
-                entityManager.persist(instance);
-            });
-            const meta = new meta_entity.RegistryMeta(currMeta);
-            entityManager.persist(meta);
-        });
+        await this.save(updatedKeys, currentOperators, currMeta);
         this.logger.log('Saved data to the DB', {
             operators: currentOperators.length,
             updatedKeys: updatedKeys.length,
@@ -156,8 +148,12 @@ exports.AbstractRegistryService = class AbstractRegistryService {
             const unchangedKeysMaxIndex = isSameOperator
                 ? prevOperator.usedSigningKeys
                 : 0;
-            const fromIndex = unchangedKeysMaxIndex;
+            // get the right border up to which the keys should be updated
+            // it's different for different scenarios
             const toIndex = this.getToIndex(currOperator);
+            // fromIndex may become larger than toIndex if used keys are deleted
+            // this should not happen in mainnet, but sometimes keys can be deleted in testnet by modification of the contract
+            const fromIndex = unchangedKeysMaxIndex <= toIndex ? unchangedKeysMaxIndex : 0;
             const operatorIndex = currOperator.index;
             const overrides = { blockTag: { blockHash } };
             const result = await this.keyFetch.fetch(operatorIndex, fromIndex, toIndex, overrides);
@@ -169,7 +165,7 @@ exports.AbstractRegistryService = class AbstractRegistryService {
             });
             return result;
         }));
-        return keysByOperator.flat();
+        return keysByOperator.flat().filter((key) => key);
     }
     /** storage */
     /** returns the latest meta data from the db */
@@ -180,6 +176,48 @@ exports.AbstractRegistryService = class AbstractRegistryService {
     async getOperatorsFromStorage() {
         return await this.operatorStorage.findAll();
     }
+    /** returns all the keys from storage */
+    async getOperatorsKeysFromStorage() {
+        return await this.keyStorage.findAll();
+    }
+    /** saves all the data to the db */
+    async save(updatedKeys, currentOperators, currMeta) {
+        // save all data in a transaction
+        await this.entityManager.transactional(async (entityManager) => {
+            await Promise.all(
+            // remove all keys from the database that are greater than the total number of keys
+            // it's needed to clear the list in db when removing keys from the contract
+            currentOperators.map(async (operator) => {
+                await entityManager.nativeDelete(key_entity.RegistryKey, {
+                    index: { $gte: operator.totalSigningKeys },
+                    operatorIndex: operator.index,
+                });
+            }));
+            await Promise.all(
+            // 500 — SQLite limit in insert operation
+            utils.chunk(updatedKeys, 499).map(async (keysChunk) => {
+                await entityManager
+                    .createQueryBuilder(key_entity.RegistryKey)
+                    .insert(keysChunk)
+                    .onConflict(['index', 'operator_index'])
+                    .merge()
+                    .execute();
+            }));
+            await Promise.all(
+            // 500 — SQLite limit in insert operation
+            utils.chunk(currentOperators, 499).map(async (operatorsChunk) => {
+                await entityManager
+                    .createQueryBuilder(operator_entity.RegistryOperator)
+                    .insert(operatorsChunk)
+                    .onConflict('index')
+                    .merge()
+                    .execute();
+            }));
+            // replace metadata with new one
+            await entityManager.nativeDelete(meta_entity.RegistryMeta, {});
+            await entityManager.persist(new meta_entity.RegistryMeta(currMeta));
+        });
+    }
     /** clears the db */
     async clear() {
         await this.entityManager.transactional(async (entityManager) => {
@@ -207,5 +245,5 @@ exports.AbstractRegistryService = tslib.__decorate([
         key_storage.RegistryKeyStorageService,
         operator_fetch.RegistryOperatorFetchService,
         operator_storage.RegistryOperatorStorageService,
-        core.EntityManager, Object])
+        knex.EntityManager, Object])
 ], exports.AbstractRegistryService);

package/dist/storage/key.repository.d.ts

@@ -1,4 +1,4 @@
-import { EntityRepository } from '@mikro-orm/core';
+import { EntityRepository } from '@mikro-orm/knex';
 import { RegistryKey } from './key.entity';
 export declare class RegistryKeyRepository extends EntityRepository<RegistryKey> {
 }

package/dist/storage/key.repository.js

@@ -2,9 +2,9 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-var core = require('@mikro-orm/core');
+var knex = require('@mikro-orm/knex');
 
-class RegistryKeyRepository extends core.EntityRepository {
+class RegistryKeyRepository extends knex.EntityRepository {
 }
 
 exports.RegistryKeyRepository = RegistryKeyRepository;

package/dist/storage/key.storage.js

@@ -3,6 +3,7 @@
 Object.defineProperty(exports, '__esModule', { value: true });
 
 var tslib = require('tslib');
+var core = require('@mikro-orm/core');
 var common = require('@nestjs/common');
 var key_entity = require('./key.entity.js');
 var key_repository = require('./key.repository.js');
@@ -13,7 +14,9 @@ exports.RegistryKeyStorageService = class RegistryKeyStorageService {
     }
     /** find all keys */
     async findAll() {
-        return await this.repository.findAll();
+        return await this.repository.findAll({
+            orderBy: [{ operatorIndex: core.QueryOrder.ASC }, { index: core.QueryOrder.ASC }],
+        });
     }
     /** find used keys */
     async findUsed() {

package/dist/storage/meta.repository.d.ts

@@ -1,4 +1,4 @@
-import { EntityRepository } from '@mikro-orm/core';
+import { EntityRepository } from '@mikro-orm/knex';
 import { RegistryMeta } from './meta.entity';
 export declare class RegistryMetaRepository extends EntityRepository<RegistryMeta> {
 }

package/dist/storage/meta.repository.js

@@ -2,9 +2,9 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-var core = require('@mikro-orm/core');
+var knex = require('@mikro-orm/knex');
 
-class RegistryMetaRepository extends core.EntityRepository {
+class RegistryMetaRepository extends knex.EntityRepository {
 }
 
 exports.RegistryMetaRepository = RegistryMetaRepository;

package/dist/storage/operator.repository.d.ts

@@ -1,4 +1,4 @@
-import { EntityRepository } from '@mikro-orm/core';
+import { EntityRepository } from '@mikro-orm/knex';
 import { RegistryOperator } from './operator.entity';
 export declare class RegistryOperatorRepository extends EntityRepository<RegistryOperator> {
 }

package/dist/storage/operator.repository.js

@@ -2,9 +2,9 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-var core = require('@mikro-orm/core');
+var knex = require('@mikro-orm/knex');
 
-class RegistryOperatorRepository extends core.EntityRepository {
+class RegistryOperatorRepository extends knex.EntityRepository {
 }
 
 exports.RegistryOperatorRepository = RegistryOperatorRepository;

package/dist/storage/operator.storage.js

@@ -3,6 +3,7 @@
 Object.defineProperty(exports, '__esModule', { value: true });
 
 var tslib = require('tslib');
+var core = require('@mikro-orm/core');
 var common = require('@nestjs/common');
 var operator_entity = require('./operator.entity.js');
 var operator_repository = require('./operator.repository.js');
@@ -13,7 +14,9 @@ exports.RegistryOperatorStorageService = class RegistryOperatorStorageService {
     }
     /** find all operators */
     async findAll() {
-        return await this.repository.findAll();
+        return await this.repository.findAll({
+            orderBy: [{ index: core.QueryOrder.ASC }],
+        });
     }
     /** find operator by index */
     async findOneByIndex(operatorIndex) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lido-nestjs/registry",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
@@ -31,11 +31,12 @@
     "@lido-nestjs/contracts": "2.1.0",
     "@lido-nestjs/decorators": "1.0.0",
     "@lido-nestjs/logger": "1.1.0",
-    "@lido-nestjs/utils": "1.1.0",
+    "@lido-nestjs/utils": "1.2.0",
     "cron": "^2.0.0"
   },
   "peerDependencies": {
     "@mikro-orm/core": "^5.2.0",
+    "@mikro-orm/knex": "^5.2.0",
     "@mikro-orm/nestjs": "^5.0.2",
     "@nestjs/common": "^8.2.5",
     "@nestjs/core": "^8.2.5",
@@ -46,6 +47,7 @@
   "devDependencies": {
     "@lido-nestjs/execution": "1.7.0",
     "@mikro-orm/core": "^5.2.0",
+    "@mikro-orm/knex": "^5.2.0",
     "@mikro-orm/nestjs": "^5.0.2",
     "@mikro-orm/sqlite": "^5.2.0",
     "@nestjs/common": "^8.2.5",