@dashevo/dapi-client 0.24.0-dev.6 → 0.25.0-dev.2

package/lib/BlockHeadersProvider/BlockHeadersReader.js

@@ -1,4 +1,6 @@
 const { EventEmitter } = require('events');
+const { BlockHeader } = require('@dashevo/dashcore-lib');
+const GrpcErrorCodes = require('@dashevo/grpc-common/lib/server/error/GrpcErrorCodes');
 
 const EVENTS = {
   BLOCK_HEADERS: 'BLOCK_HEADERS',
@@ -6,15 +8,8 @@ const EVENTS = {
   ERROR: 'error',
 };
 
-const COMMANDS = {
-  HANDLE_FINISHED_STREAM: 'HANDLE_FINISHED_STREAM',
-  HANDLE_STREAM_RETRY: 'HANDLE_STREAM_RETRY',
-  HANDLE_STREAM_ERROR: 'HANDLE_STREAM_ERROR',
-};
-
 /**
  * @typedef BlockHeadersReaderOptions
- * @property {CoreMethodsFacade} [coreMethods]
  * @property {number} [maxParallelStreams]
  * @property {number} [targetBatchSize]
  * @property {number} [maxRetries]
@@ -23,10 +18,13 @@ const COMMANDS = {
 class BlockHeadersReader extends EventEmitter {
   /**
    * @param {BlockHeadersReaderOptions} options
+   * @param createHistoricalSyncStream
+   * @param createContinuousSyncStream
    */
-  constructor(options = {}) {
+  constructor(options = {}, createHistoricalSyncStream, createContinuousSyncStream) {
     super();
-    this.coreMethods = options.coreMethods;
+    this.createHistoricalSyncStream = createHistoricalSyncStream;
+    this.createContinuousSyncStream = createContinuousSyncStream;
     this.maxParallelStreams = options.maxParallelStreams;
     this.targetBatchSize = options.targetBatchSize;
     this.maxRetries = options.maxRetries;
@@ -34,9 +32,16 @@ class BlockHeadersReader extends EventEmitter {
     /**
      * Holds references to the historical streams
      *
-     * @type {*[]}
+     * @type {Stream[]}
      */
     this.historicalStreams = [];
+
+    /**
+     * Holds reference to the continuous sync stream
+     *
+     * @type {Stream}
+     */
+    this.continuousSyncStream = null;
   }
 
   /**
@@ -48,52 +53,30 @@ class BlockHeadersReader extends EventEmitter {
    */
   async readHistorical(fromBlockHeight, toBlockHeight) {
     if (this.historicalStreams.length) {
-      throw new Error('Historical streams are already running');
+      throw new Error('Historical streams are already running. Please stop them first.');
     }
 
-    const totalAmount = toBlockHeight - fromBlockHeight + 1;
-    if (totalAmount === 0) {
-      return;
+    if (fromBlockHeight <= 0) {
+      throw new Error(`Invalid fromBlockHeight value: ${fromBlockHeight}`);
     }
 
-    if (totalAmount < 0) {
+    const totalAmount = toBlockHeight - fromBlockHeight + 1;
+    if (totalAmount <= 0) {
       throw new Error(`Invalid total amount of headers to read: ${totalAmount}`);
     }
 
-    // Resubscribe to the stream in case of error, and replace the stream in the array
-    this.on(COMMANDS.HANDLE_STREAM_RETRY, (oldStream, newStream) => {
-      const index = this.historicalStreams.indexOf(oldStream);
-      this.historicalStreams[index] = newStream;
-    });
-
-    // Remove stream from the array in case of error
-    this.on(COMMANDS.HANDLE_STREAM_ERROR, (stream, e) => {
-      const index = this.historicalStreams.indexOf(stream);
-      this.historicalStreams.splice(index, 1);
-      this.emit(EVENTS.ERROR, e);
-    });
-
-    // Remove finished stream from the array and emit HISTORICAL_DATA_OBTAINED event
-    this.on(COMMANDS.HANDLE_FINISHED_STREAM, (stream) => {
-      const index = this.historicalStreams.indexOf(stream);
-      this.historicalStreams.splice(index, 1);
-      if (this.historicalStreams.length === 0) {
-        this.emit(EVENTS.HISTORICAL_DATA_OBTAINED);
-      }
-    });
-
     const numStreams = Math.min(
       Math.max(Math.round(totalAmount / this.targetBatchSize), 1),
       this.maxParallelStreams,
     );
 
     const actualBatchSize = Math.ceil(totalAmount / numStreams);
+
     for (let batchIndex = 0; batchIndex < numStreams; batchIndex += 1) {
-      const startingHeight = (batchIndex * actualBatchSize) + 1;
+      const startingHeight = (batchIndex * actualBatchSize) + fromBlockHeight;
       const count = Math.min(actualBatchSize, toBlockHeight - startingHeight + 1);
 
-      const subscribeWithRetries = this.subscribeToHistoricalBatch(this.maxRetries);
-
+      const subscribeWithRetries = this.createSubscribeToHistoricalBatch(this.maxRetries);
       // eslint-disable-next-line no-await-in-loop
       const stream = await subscribeWithRetries(startingHeight, count);
       this.historicalStreams.push(stream);
@@ -101,48 +84,106 @@ class BlockHeadersReader extends EventEmitter {
   }
 
   stopReadingHistorical() {
-    this.removeAllListeners(COMMANDS.HANDLE_STREAM_RETRY);
-    this.removeAllListeners(COMMANDS.HANDLE_STREAM_ERROR);
-    this.removeAllListeners(COMMANDS.HANDLE_FINISHED_STREAM);
-    this.historicalStreams.forEach((stream) => stream.destroy());
-    this.historicalStreams = [];
+    while (this.historicalStreams.length) {
+      const stream = this.historicalStreams.shift();
+      this.cancelStream(stream);
+    }
   }
 
   /**
    * Subscribes to continuously arriving block headers
    *
    * @param {number} fromBlockHeight
-   * @returns {Promise<Stream>}
+   * @returns {Promise<ReconnectableStream>}
    */
   async subscribeToNew(fromBlockHeight) {
-    const stream = await this.coreMethods.subscribeToBlockHeadersWithChainLocks({
-      fromBlockHeight,
-    });
+    if (this.continuousSyncStream) {
+      throw new Error('Continuous sync has already been started');
+    }
+
+    if (typeof fromBlockHeight !== 'number' || fromBlockHeight < 1) {
+      throw new Error(`Invalid fromBlockHeight: ${fromBlockHeight}`);
+    }
 
-    stream.on('data', (data) => {
-      const blockHeaders = data.getBlockHeaders();
+    // We don't know yet whether we already have header at fromBlockHeight
+    let lastKnownChainHeight = fromBlockHeight - 1;
+
+    const stream = await this.createContinuousSyncStream(fromBlockHeight);
+
+    const errorHandler = (e) => {
+      this.continuousSyncStream = null;
+      if (e.code === GrpcErrorCodes.CANCELLED) {
+        return;
+      }
+      this.emit(EVENTS.ERROR, e);
+    };
+
+    const dataHandler = (data) => {
+      const blockHeadersResponse = data.getBlockHeaders();
+
+      if (blockHeadersResponse) {
+        const rawHeaders = blockHeadersResponse.getHeadersList();
+
+        const headers = rawHeaders.map((header) => new BlockHeader(Buffer.from(header)));
+
+        lastKnownChainHeight += headers.length;
+        const batchHeadHeight = lastKnownChainHeight - headers.length + 1;
 
-      if (blockHeaders) {
         /**
          * Kills stream in case of deliberate rejection from the outside
          *
          * @param e
          */
-        const rejectHeaders = (e) => {
-          stream.destroy(e);
+        const rejectHeaders = async (e) => {
+          // Don't use cancelStream there because it's going to unsubscribe from events
+          stream.cancel();
+          stream.retryOnError(e);
         };
 
-        this.emit(EVENTS.BLOCK_HEADERS, blockHeaders.getHeadersList(), rejectHeaders);
+        this.emit(EVENTS.BLOCK_HEADERS, {
+          headers,
+          headHeight: batchHeadHeight,
+        }, rejectHeaders);
       }
-    });
+    };
 
-    stream.on('error', (e) => {
-      this.emit(EVENTS.ERROR, e);
-    });
+    const beforeReconnectHandler = (updateArguments) => {
+      updateArguments({
+        fromBlockHeight: lastKnownChainHeight,
+        count: 0,
+      });
+
+      lastKnownChainHeight -= 1;
+    };
 
+    const endHandler = () => {
+      this.continuousSyncStream = null;
+    };
+
+    stream.on('data', dataHandler);
+    stream.on('beforeReconnect', beforeReconnectHandler);
+    stream.on('error', errorHandler);
+    stream.on('end', endHandler);
+
+    stream.removeAllListeners = () => {
+      stream.removeListener('data', dataHandler);
+      stream.removeListener('end', endHandler);
+      stream.removeListener('beforeReconnect', beforeReconnectHandler);
+    };
+
+    this.continuousSyncStream = stream;
     return stream;
   }
 
+  unsubscribeFromNew() {
+    if (this.continuousSyncStream) {
+      this.cancelStream(this.continuousSyncStream);
+      this.continuousSyncStream = null;
+    }
+  }
+
+  // TODO: refactor whole thing with ReconnectableStream that supports
+  // retry on error logic
   /**
    * A HOF that returns a function to subscribe to historical block headers and chain locks
    * and handles retry logic
@@ -151,7 +192,7 @@ class BlockHeadersReader extends EventEmitter {
    * @param {number} [maxRetries=0] - maximum amount of retries
    * @returns {function(*, *): Promise<Stream>}
    */
-  subscribeToHistoricalBatch(maxRetries = 0) {
+  createSubscribeToHistoricalBatch(maxRetries = 0) {
     let currentRetries = 0;
 
     /**
@@ -164,16 +205,45 @@ class BlockHeadersReader extends EventEmitter {
     const subscribeWithRetries = async (fromBlockHeight, count) => {
       let headersObtained = 0;
 
-      const stream = await this.coreMethods.subscribeToBlockHeadersWithChainLocks({
-        fromBlockHeight,
-        count,
-      });
+      const stream = await this.createHistoricalSyncStream(fromBlockHeight, count);
+
+      const errorHandler = (streamError) => {
+        if (streamError.code === GrpcErrorCodes.CANCELLED) {
+          // TODO: consider reworking with COMMANDS instead
+          // of producing a side effect that alters class state
+          const index = this.historicalStreams.indexOf(stream);
+          if (index >= 0) {
+            this.historicalStreams.splice(index, 1);
+          }
+          return;
+        }
+
+        if (currentRetries < maxRetries) {
+          const newFromBlockHeight = fromBlockHeight + headersObtained;
+          const newCount = count - headersObtained;
+
+          // TODO: do not retry in case newCount is zero
+          subscribeWithRetries(newFromBlockHeight, newCount)
+            .then((newStream) => {
+              const index = this.historicalStreams.indexOf(stream);
+              this.historicalStreams[index] = newStream;
+              currentRetries += 1;
+            }).catch((e) => {
+              this.stopReadingHistorical();
+              this.emit(EVENTS.ERROR, e);
+            });
+        } else {
+          this.stopReadingHistorical();
+          this.emit(EVENTS.ERROR, streamError);
+        }
+      };
 
-      stream.on('data', (data) => {
+      const dataHandler = (data) => {
         const blockHeaders = data.getBlockHeaders();
 
         if (blockHeaders) {
-          const headersList = blockHeaders.getHeadersList();
+          const headersList = blockHeaders.getHeadersList()
+            .map((header) => new BlockHeader(Buffer.from(header)));
 
           let rejected = false;
 
@@ -184,46 +254,58 @@ class BlockHeadersReader extends EventEmitter {
            */
           const rejectHeaders = (e) => {
             rejected = true;
-            stream.destroy(e);
+            // Cancel stream and unsubscribe from all data events
+            // because they are going th be re-created in errorHandler
+            this.cancelStream(stream);
+            errorHandler(e);
           };
 
-          this.emit(EVENTS.BLOCK_HEADERS, headersList, rejectHeaders);
+          const batchHeadHeight = fromBlockHeight + headersObtained;
+          this.emit(EVENTS.BLOCK_HEADERS, {
+            headers: headersList,
+            headHeight: batchHeadHeight,
+          }, rejectHeaders);
 
           if (!rejected) {
             headersObtained += headersList.length;
           }
         }
-      });
+      };
 
-      stream.on('error', (streamError) => {
-        if (currentRetries < maxRetries) {
-          const newFromBlockHeight = fromBlockHeight + headersObtained;
-          const newCount = count - headersObtained;
-
-          subscribeWithRetries(newFromBlockHeight, newCount)
-            .then((newStream) => {
-              currentRetries += 1;
-              this.emit(COMMANDS.HANDLE_STREAM_RETRY, stream, newStream);
-            }).catch((e) => {
-              this.emit(COMMANDS.HANDLE_STREAM_ERROR, stream, e);
-            });
-        } else {
-          this.emit(COMMANDS.HANDLE_STREAM_ERROR, stream, streamError);
+      const endHandler = () => {
+        const index = this.historicalStreams.indexOf(stream);
+        this.historicalStreams.splice(index, 1);
+        if (this.historicalStreams.length === 0) {
+          this.emit(EVENTS.HISTORICAL_DATA_OBTAINED);
         }
-      });
+      };
 
-      stream.on('end', () => {
-        this.emit(COMMANDS.HANDLE_FINISHED_STREAM, stream);
-      });
+      // TODO: consider reworking with "pasued" mode to
+      // control the data flow manually because SPV verification is
+      // resource intense and backpressure could be a problem
+      // also the whole thing severely affects UI responsiveness
+      stream.on('data', dataHandler);
+      stream.on('error', errorHandler);
+      stream.on('end', endHandler);
+
+      stream.removeAllListeners = () => {
+        stream.removeListener('data', dataHandler);
+        stream.removeListener('end', endHandler);
+      };
 
       return stream;
     };
 
     return subscribeWithRetries;
   }
+
+  // eslint-disable-next-line class-methods-use-this
+  cancelStream(stream) {
+    stream.removeAllListeners();
+    stream.cancel();
+  }
 }
 
 BlockHeadersReader.EVENTS = EVENTS;
-BlockHeadersReader.COMMANDS = COMMANDS;
 
 module.exports = BlockHeadersReader;

package/lib/BlockHeadersProvider/createBlockHeadersProviderFromOptions.js

@@ -1,6 +1,6 @@
-const networks = require('@dashevo/dashcore-lib/lib/networks');
 const DAPIClientError = require('../errors/DAPIClientError');
 const BlockHeadersProvider = require('./BlockHeadersProvider');
+const ReconnectableStream = require('../transport/ReconnectableStream');
 
 const validateNumber = (value, name, min = NaN, max = NaN) => {
   if (typeof value !== 'number') {
@@ -11,7 +11,7 @@ const validateNumber = (value, name, min = NaN, max = NaN) => {
     throw new DAPIClientError(`'${name}' can not be less than ${min}`);
   }
 
-  if (!Number.isNaN(max) && value > min) {
+  if (!Number.isNaN(max) && value > max) {
     throw new DAPIClientError(`'${name}' can not be more than ${max}`);
   }
 };
@@ -32,45 +32,60 @@ function createBlockHeadersProviderFromOptions(options, coreMethods) {
     blockHeadersProvider = options.blockHeadersProvider;
   }
 
+  const createContinuousSyncStream = (fromBlockHeight) => ReconnectableStream
+    .create(
+      coreMethods.subscribeToBlockHeadersWithChainLocks,
+      {
+        maxRetriesOnError: -1,
+      },
+    )({
+      fromBlockHeight,
+    });
+
+  const createHistoricalSyncStream = (fromBlockHeight, count) => {
+    const { subscribeToBlockHeadersWithChainLocks } = coreMethods;
+    return subscribeToBlockHeadersWithChainLocks({
+      fromBlockHeight,
+      count,
+    });
+  };
+
   if (options.blockHeadersProviderOptions) {
+    const { network } = options;
+
     const blockHeadersProviderOptions = {
       ...BlockHeadersProvider.defaultOptions,
       ...options.blockHeadersProviderOptions,
+      network,
     };
 
     const {
-      network,
-      autoStart,
       maxParallelStreams,
       targetBatchSize,
       fromBlockHeight,
       maxRetries,
     } = blockHeadersProviderOptions;
 
-    if (network && !networks.get(network)) {
-      throw new DAPIClientError(`Invalid network '${options.network}'`);
-    }
-
-    if (typeof autoStart !== 'boolean') {
-      throw new DAPIClientError('\'autoStart\' option must have boolean type');
-    }
-
     validateNumber(maxParallelStreams, 'maxParallelStreams', 1);
     validateNumber(targetBatchSize, 'targetBatchSize', 1);
     validateNumber(fromBlockHeight, 'fromBlockHeight', 1);
-    validateNumber(maxRetries, 'maxRetries', 0);
+    validateNumber(maxRetries, 'maxRetries', 0, 100);
 
     blockHeadersProvider = new BlockHeadersProvider(
       blockHeadersProviderOptions,
+      createHistoricalSyncStream,
+      createContinuousSyncStream,
     );
   }
 
   if (!blockHeadersProvider) {
-    blockHeadersProvider = new BlockHeadersProvider();
+    blockHeadersProvider = new BlockHeadersProvider(
+      {},
+      createContinuousSyncStream,
+      createHistoricalSyncStream,
+    );
   }
 
-  blockHeadersProvider.setCoreMethods(coreMethods);
-
   return blockHeadersProvider;
 }
 

package/lib/DAPIClient.js

@@ -65,12 +65,6 @@ class DAPIClient extends EventEmitter {
     this.blockHeadersProvider.on(BlockHeadersProvider.EVENTS.ERROR, (e) => {
       this.emit(EVENTS.ERROR, e);
     });
-
-    if (this.options.blockHeadersProviderOptions.autoStart) {
-      this.blockHeadersProvider.start().catch((e) => {
-        this.emit(EVENTS.ERROR, e);
-      });
-    }
   }
 }
 

package/lib/dapiAddressProvider/DAPIAddress.js

@@ -10,13 +10,12 @@ class DAPIAddress {
     }
 
     if (typeof address === 'string') {
-      const [host, httpPort, grpcPort, ssl] = address.split(':');
+      const [host, port, ssl] = address.split(':');
 
       // eslint-disable-next-line no-param-reassign
       address = {
         host,
-        httpPort: httpPort ? parseInt(httpPort, 10) : DAPIAddress.DEFAULT_HTTP_PORT,
-        grpcPort: grpcPort ? parseInt(grpcPort, 10) : DAPIAddress.DEFAULT_GRPC_PORT,
+        port: port ? parseInt(port, 10) : DAPIAddress.DEFAULT_PORT,
         protocol: ssl === 'no-ssl' ? 'http' : DAPIAddress.DEFAULT_PROTOCOL,
         allowSelfSignedCertificate: ssl === 'self-signed',
       };
@@ -28,8 +27,7 @@ class DAPIAddress {
 
     this.protocol = address.protocol || DAPIAddress.DEFAULT_PROTOCOL;
     this.host = address.host;
-    this.httpPort = address.httpPort || DAPIAddress.DEFAULT_HTTP_PORT;
-    this.grpcPort = address.grpcPort || DAPIAddress.DEFAULT_GRPC_PORT;
+    this.port = address.port || DAPIAddress.DEFAULT_PORT;
     this.proRegTxHash = address.proRegTxHash;
     this.allowSelfSignedCertificate = address.allowSelfSignedCertificate || false;
 
@@ -68,43 +66,22 @@ class DAPIAddress {
   }
 
   /**
-   * Get HTTP port
+   * Get port
    *
    * @returns {number}
    */
-  getHttpPort() {
-    return this.httpPort;
+  getPort() {
+    return this.port;
   }
 
   /**
-   * Set HTTP port
+   * Set port
    *
    * @param {number} port
    * @returns {DAPIAddress}
    */
-  setHttpPort(port) {
-    this.httpPort = port;
-
-    return this;
-  }
-
-  /**
-   * Get gRPC port
-   *
-   * @returns {number}
-   */
-  getGrpcPort() {
-    return this.grpcPort;
-  }
-
-  /**
-   * Set gRPC port
-   *
-   * @param {number} port
-   * @returns {DAPIAddress}
-   */
-  setGrpcPort(port) {
-    this.grpcPort = port;
+  setPort(port) {
+    this.port = port;
 
     return this;
   }
@@ -179,28 +156,25 @@ class DAPIAddress {
     return {
       protocol: this.getProtocol(),
       host: this.getHost(),
-      httpPort: this.getHttpPort(),
-      grpcPort: this.getGrpcPort(),
+      port: this.getPort(),
       proRegTxHash: this.getProRegTxHash(),
       allowSelfSignedCertificate: this.isSelfSignedCertificateAllowed(),
     };
   }
 
   toString() {
-    return `${this.getProtocol()}://${this.getHost()}:${this.getHttpPort()}:${this.getGrpcPort()}`;
+    return `${this.getProtocol()}://${this.getHost()}:${this.getPort()}`;
   }
 }
 
-DAPIAddress.DEFAULT_HTTP_PORT = 3000;
-DAPIAddress.DEFAULT_GRPC_PORT = 3010;
-DAPIAddress.DEFAULT_PROTOCOL = 'http';
+DAPIAddress.DEFAULT_PORT = 443;
+DAPIAddress.DEFAULT_PROTOCOL = 'https';
 
 /**
  * @typedef {object} RawDAPIAddress
  * @property {string} protocol
  * @property {string} host
- * @property {number} [httpPort=3000]
- * @property {number} [grpcPort=3010]
+ * @property {number} [port=443]
  * @property {string} [proRegTxHash]
  * @property {bool} [selfSigned]
  */

package/lib/dapiAddressProvider/ListDAPIAddressProvider.js

@@ -1,4 +1,4 @@
-const sample = require('lodash.sample');
+const sample = require('lodash/sample');
 const networks = require('@dashevo/dashcore-lib/lib/networks');
 
 class ListDAPIAddressProvider {
@@ -30,7 +30,7 @@ class ListDAPIAddressProvider {
     }
 
     // This is a temporary fix for a localhost masternode.
-    // On mac os, internal docker IP is used to register masternode, and it's
+    // On macOS, internal docker IP is used to register masternode, and it's
     // not really possible to bind to that address, so that workaround is introduced.
     const network = networks.get(this.options.network);
     if (network && network.regtestEnabled) {
@@ -39,8 +39,7 @@ class ListDAPIAddressProvider {
       liveAddress.protocol = 'https';
       liveAddress.host = '127.0.0.1';
       liveAddress.allowSelfSignedCertificate = true;
-      liveAddress.httpPort = 3000 + randomNodeIndex * 100;
-      liveAddress.grpcPort = 3010 + randomNodeIndex * 100;
+      liveAddress.port = 2443 + randomNodeIndex * 100;
     }
 
     return liveAddress;

package/lib/dapiAddressProvider/SimplifiedMasternodeListDAPIAddressProvider.js

@@ -19,10 +19,15 @@ class SimplifiedMasternodeListDAPIAddressProvider {
    */
   async getLiveAddress() {
     const sml = await this.smlProvider.getSimplifiedMNList();
-    const validMasternodeList = sml.getValidMasternodesList();
+    const validMasternodeList = sml.getValidMasternodesList()
+      // Keep only HP masternodes
+      .filter((smlEntry) => smlEntry.nType === 1);
 
     const addressesByRegProTxHashes = {};
+    let allowSelfSignedCertificate;
     this.listDAPIAddressProvider.getAllAddresses().forEach((address) => {
+      allowSelfSignedCertificate = address.isSelfSignedCertificateAllowed();
+
       if (!address.getProRegTxHash()) {
         return;
       }
@@ -36,6 +41,8 @@ class SimplifiedMasternodeListDAPIAddressProvider {
       if (!address) {
         address = new DAPIAddress({
           host: smlEntry.getIp(),
+          port: smlEntry.platformHTTPPort,
+          allowSelfSignedCertificate,
           proRegTxHash: smlEntry.proRegTxHash,
         });
       } else {

package/lib/index.js

@@ -1,9 +1,14 @@
+require('../polyfills/fetch-polyfill');
+
 const DAPIClient = require('./DAPIClient');
 
 const NotFoundError = require('./transport/GrpcTransport/errors/NotFoundError');
+const BlockHeadersProvider = require('./BlockHeadersProvider/BlockHeadersProvider');
 
 DAPIClient.Errors = {
   NotFoundError,
 };
 
+DAPIClient.BlockHeadersProvider = BlockHeadersProvider;
+
 module.exports = DAPIClient;