@dashevo/dapi-client 0.23.0 → 0.23.1

package/lib/BlockHeadersProvider/BlockHeadersProvider.js

@@ -205,6 +205,10 @@ class BlockHeadersProvider extends EventEmitter {
   headersHandler(headersData, reject) {
     const { headers, headHeight } = headersData;
 
+    // TODO: simulate headers rejection and debug
+    // BlockHeadersReader might not correctly set startBlockHeight
+    // which results in orphan chunks in SPV
+
     try {
       const headersAdded = this.spvChain.addHeaders(headers, headHeight);
 

package/lib/BlockHeadersProvider/BlockHeadersReader.js

@@ -86,7 +86,7 @@ class BlockHeadersReader extends EventEmitter {
   stopReadingHistorical() {
     while (this.historicalStreams.length) {
       const stream = this.historicalStreams.shift();
-      stream.cancel();
+      this.cancelStream(stream);
     }
   }
 
@@ -105,11 +105,20 @@ class BlockHeadersReader extends EventEmitter {
       throw new Error(`Invalid fromBlockHeight: ${fromBlockHeight}`);
     }
 
+    // We don't know yet whether we already have header at fromBlockHeight
     let lastKnownChainHeight = fromBlockHeight - 1;
 
     const stream = await this.createContinuousSyncStream(fromBlockHeight);
 
-    stream.on('data', (data) => {
+    const errorHandler = (e) => {
+      this.continuousSyncStream = null;
+      if (e.code === GrpcErrorCodes.CANCELLED) {
+        return;
+      }
+      this.emit(EVENTS.ERROR, e);
+    };
+
+    const dataHandler = (data) => {
       const blockHeadersResponse = data.getBlockHeaders();
 
       if (blockHeadersResponse) {
@@ -125,35 +134,42 @@ class BlockHeadersReader extends EventEmitter {
          *
          * @param e
          */
-        const rejectHeaders = (e) => {
-          stream.destroy(e);
+        const rejectHeaders = async (e) => {
+          // Don't use cancelStream there because it's going to unsubscribe from events
+          stream.cancel();
+          stream.retryOnError(e);
         };
+
         this.emit(EVENTS.BLOCK_HEADERS, {
           headers,
           headHeight: batchHeadHeight,
         }, rejectHeaders);
       }
-    });
+    };
 
-    stream.on('beforeReconnect', (updateArguments) => {
+    const beforeReconnectHandler = (updateArguments) => {
       updateArguments({
         fromBlockHeight: lastKnownChainHeight,
+        count: 0,
       });
 
       lastKnownChainHeight -= 1;
-    });
+    };
 
-    stream.on('error', (e) => {
+    const endHandler = () => {
       this.continuousSyncStream = null;
-      if (e.code === GrpcErrorCodes.CANCELLED) {
-        return;
-      }
-      this.emit(EVENTS.ERROR, e);
-    });
+    };
 
-    stream.on('end', () => {
-      this.continuousSyncStream = null;
-    });
+    stream.on('data', dataHandler);
+    stream.on('beforeReconnect', beforeReconnectHandler);
+    stream.on('error', errorHandler);
+    stream.on('end', endHandler);
+
+    stream.removeAllListeners = () => {
+      stream.removeListener('data', dataHandler);
+      stream.removeListener('end', endHandler);
+      stream.removeListener('beforeReconnect', beforeReconnectHandler);
+    };
 
     this.continuousSyncStream = stream;
     return stream;
@@ -161,11 +177,13 @@ class BlockHeadersReader extends EventEmitter {
 
   unsubscribeFromNew() {
     if (this.continuousSyncStream) {
-      this.continuousSyncStream.cancel();
+      this.cancelStream(this.continuousSyncStream);
       this.continuousSyncStream = null;
     }
   }
 
+  // TODO: refactor whole thing with ReconnectableStream that supports
+  // retry on error logic
   /**
    * A HOF that returns a function to subscribe to historical block headers and chain locks
    * and handles retry logic
@@ -189,37 +207,6 @@ class BlockHeadersReader extends EventEmitter {
 
       const stream = await this.createHistoricalSyncStream(fromBlockHeight, count);
 
-      const dataHandler = (data) => {
-        const blockHeaders = data.getBlockHeaders();
-
-        if (blockHeaders) {
-          const headersList = blockHeaders.getHeadersList()
-            .map((header) => new BlockHeader(Buffer.from(header)));
-
-          let rejected = false;
-
-          /**
-           * Kills stream in case of deliberate rejection from the outside
-           *
-           * @param e
-           */
-          const rejectHeaders = (e) => {
-            rejected = true;
-            stream.destroy(e);
-          };
-
-          const batchHeadHeight = fromBlockHeight + headersObtained;
-          this.emit(EVENTS.BLOCK_HEADERS, {
-            headers: headersList,
-            headHeight: batchHeadHeight,
-          }, rejectHeaders);
-
-          if (!rejected) {
-            headersObtained += headersList.length;
-          }
-        }
-      };
-
       const errorHandler = (streamError) => {
         if (streamError.code === GrpcErrorCodes.CANCELLED) {
           // TODO: consider reworking with COMMANDS instead
@@ -251,6 +238,40 @@ class BlockHeadersReader extends EventEmitter {
         }
       };
 
+      const dataHandler = (data) => {
+        const blockHeaders = data.getBlockHeaders();
+
+        if (blockHeaders) {
+          const headersList = blockHeaders.getHeadersList()
+            .map((header) => new BlockHeader(Buffer.from(header)));
+
+          let rejected = false;
+
+          /**
+           * Kills stream in case of deliberate rejection from the outside
+           *
+           * @param e
+           */
+          const rejectHeaders = (e) => {
+            rejected = true;
+            // Cancel stream and unsubscribe from all data events
+            // because they are going th be re-created in errorHandler
+            this.cancelStream(stream);
+            errorHandler(e);
+          };
+
+          const batchHeadHeight = fromBlockHeight + headersObtained;
+          this.emit(EVENTS.BLOCK_HEADERS, {
+            headers: headersList,
+            headHeight: batchHeadHeight,
+          }, rejectHeaders);
+
+          if (!rejected) {
+            headersObtained += headersList.length;
+          }
+        }
+      };
+
       const endHandler = () => {
         const index = this.historicalStreams.indexOf(stream);
         this.historicalStreams.splice(index, 1);
@@ -267,11 +288,22 @@ class BlockHeadersReader extends EventEmitter {
       stream.on('error', errorHandler);
       stream.on('end', endHandler);
 
+      stream.removeAllListeners = () => {
+        stream.removeListener('data', dataHandler);
+        stream.removeListener('end', endHandler);
+      };
+
       return stream;
     };
 
     return subscribeWithRetries;
   }
+
+  // eslint-disable-next-line class-methods-use-this
+  cancelStream(stream) {
+    stream.removeAllListeners();
+    stream.cancel();
+  }
 }
 
 BlockHeadersReader.EVENTS = EVENTS;

package/lib/index.js

@@ -1,3 +1,5 @@
+require('../polyfills/fetch-polyfill');
+
 const DAPIClient = require('./DAPIClient');
 
 const NotFoundError = require('./transport/GrpcTransport/errors/NotFoundError');

package/lib/test/bootstrap.js

@@ -1,3 +1,5 @@
+require('../../polyfills/fetch-polyfill');
+
 const { expect, use } = require('chai');
 const sinon = require('sinon');
 const sinonChai = require('sinon-chai');

package/lib/test/mocks/BlockHeadersWithChainLocksStreamMock.js

@@ -7,20 +7,12 @@ class BlockHeadersWithChainLocksStreamMock extends EventEmitter {
     sinon.spy(this, 'on');
     sinon.spy(this, 'removeListener');
     sinon.spy(this, 'emit');
-    sinon.spy(this, 'destroy');
     sinon.spy(this, 'removeAllListeners');
     sinon.spy(this, 'cancel');
 
     this.errored = false;
   }
 
-  destroy(e) {
-    this.emit('end');
-    if (e) {
-      this.emit('error', e);
-    }
-  }
-
   cancel() {
     if (!this.errored) {
       const err = new Error('CANCELED_ON_CLIENT');
@@ -48,6 +40,15 @@ class BlockHeadersWithChainLocksStreamMock extends EventEmitter {
       }),
     });
   }
+
+  errorHandler(e) {
+    this.emit('error', e);
+  }
+
+  end() {
+    this.emit('end');
+    this.removeAllListeners();
+  }
 }
 
 module.exports = BlockHeadersWithChainLocksStreamMock;

package/lib/transport/GrpcTransport/createGrpcTransportError.js

@@ -2,7 +2,7 @@ const cbor = require('cbor');
 
 const createConsensusError = require('@dashevo/dpp/lib/errors/consensus/createConsensusError');
 const GrpcErrorCodes = require('@dashevo/grpc-common/lib/server/error/GrpcErrorCodes');
-const { Metadata } = require('@grpc/grpc-js/build/src/metadata');
+const { parseMetadata } = require('@dashevo/dapi-grpc');
 
 const NotFoundError = require('./errors/NotFoundError');
 const TimeoutError = require('./errors/TimeoutError');
@@ -50,15 +50,7 @@ function createGrpcTransportError(grpcError, dapiAddress) {
 
   const message = grpcError.details || grpcError.message;
 
-  let metadata = {};
-
-  if (grpcError.metadata) {
-    // In cases of gRPC-Web client we get plain map instead of Metadata instance
-    metadata = grpcError.metadata;
-    if (grpcError.metadata instanceof Metadata) {
-      metadata = grpcError.metadata.getMap();
-    }
-  }
+  const metadata = parseMetadata(grpcError.metadata) || {};
 
   // Error data
   const driveErrorData = metadata['drive-error-data-bin'];

package/lib/transport/JsonRpcTransport/requestJsonRpc.js

@@ -1,5 +1,3 @@
-const axios = require('axios');
-
 const JsonRpcError = require('./errors/JsonRpcError');
 const WrongHttpCodeError = require('./errors/WrongHttpCodeError');
 
@@ -24,11 +22,6 @@ async function requestJsonRpc(host, port, method, params, options = {}) {
     id: 1,
   };
 
-  const postOptions = {};
-  if (options.timeout !== undefined) {
-    postOptions.timeout = options.timeout;
-  }
-
   const requestInfo = {
     host,
     port,
@@ -37,27 +30,32 @@ async function requestJsonRpc(host, port, method, params, options = {}) {
     options,
   };
 
-  let response;
-
-  try {
-    response = await axios.post(
-      url,
-      payload,
-      { timeout: options.timeout },
-    );
-  } catch (error) {
-    if (error.response && error.response.status >= 500) {
-      throw new WrongHttpCodeError(requestInfo, error.response.status, error.response.statusText);
-    }
+  const requestOptions = {
+    method: 'POST',
+    body: JSON.stringify(payload),
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  };
 
-    throw error;
+  let requestTimeoutId;
+  if (options.timeout) {
+    const controller = new AbortController();
+    requestTimeoutId = setTimeout(() => controller.abort(), options.timeout);
+    Object.assign(requestOptions, { signal: controller.signal });
   }
 
-  if (response.status !== 200) {
-    throw new WrongHttpCodeError(requestInfo, response.status, response.statusMessage);
+  // eslint-disable-next-line
+  const response = await fetch(url, requestOptions);
+
+  if (typeof requestTimeoutId !== 'undefined') {
+    clearTimeout(requestTimeoutId);
   }
+  const data = await response.json();
 
-  const { data } = response;
+  if (!response.ok) {
+    throw new WrongHttpCodeError(requestInfo, response.status, response.statusText);
+  }
 
   if (data.error) {
     throw new JsonRpcError(requestInfo, data.error);

package/lib/transport/ReconnectableStream.js

@@ -2,9 +2,18 @@ const EventEmitter = require('events');
 const GrpcErrorCodes = require('@dashevo/grpc-common/lib/server/error/GrpcErrorCodes');
 const wait = require('../utils/wait');
 
+/**
+ * @typedef ReconnectableStreamOptions
+ * @property {number} [autoReconnectInterval]
+ *    interval in MS to perform auto reconnect
+ * @property {number} [maxRetriesOnError]
+ *    maximum amount of retry attempts on error happens. If set to -1, retries are infinite
+ * @property {number} [retryOnErrorDelay]
+ *    delay in MS to perform retry after an error
+ */
 const defaultOptions = {
   // TODO: manage timeout according to the Nginx setting of the node
-  reconnectTimeoutDelay: 50000,
+  autoReconnectInterval: 50000,
   maxRetriesOnError: 10,
   retryOnErrorDelay: 1000,
 };
@@ -17,69 +26,121 @@ const EVENTS = {
 };
 
 /**
- * Stream that provides auto-reconnect functionality
+ * A wrapper around grpc-js/grpc-web streams that provides auto-reconnect
+ * and retry on error functionality
+ * - Auto reconnect is needed to not hang on one node for too long.
+ *   It happens within provided interval
+ * - Retry on error logic performs retry attempts until it reaches it's limit.
+ *   After reaching limit, ReconnectableStream emits an error.
  */
 class ReconnectableStream extends EventEmitter {
-  static create(fn, options) {
+  /**
+   * Helper that wraps stream creation function and performs auto connection
+   *
+   * @param {Function} createStreamFunction - function returning grpc-js/grpc-web stream
+   * @param {ReconnectableStreamOptions} options
+   * @returns {function(...[*]): Promise<ReconnectableStream>}
+   */
+  static create(createStreamFunction, options) {
     return async (...args) => {
-      const reconnectableStream = new ReconnectableStream(fn, options);
+      const reconnectableStream = new ReconnectableStream(createStreamFunction, options);
       await reconnectableStream.connect(...args);
       return reconnectableStream;
     };
   }
 
-  constructor(streamFunction, options = {}) {
+  /**
+   * @param {Function} createStream - function returning grpc-js/grpc-web stream
+   * @param {ReconnectableStreamOptions} options
+   */
+  constructor(createStream, options = {}) {
     super();
-    this.stream = null;
-    this.streamFunction = streamFunction;
-    this.reconnectTimeout = null;
 
     const opts = { ...defaultOptions, ...options };
-    this.reconnectTimeoutDelay = opts.reconnectTimeoutDelay;
-    this.maxRetriesOnError = opts.maxRetriesOnError;
+
+    /**
+     * Auto-reconnect interval in millisecond
+     * It is needed to automatically reconnect to another DAPI node
+     *
+     * @type {number}
+     */
+    this.autoReconnectInterval = opts.autoReconnectInterval;
     this.retryOnErrorDelay = opts.retryOnErrorDelay;
-    // Assign setTimeout and clearTimeout as class method for tests
-    this.setTimeout = (callback, delay) => setTimeout(callback, delay);
 
-    this.clearTimeout = (reference) => clearTimeout(reference);
+    /**
+     * Max amount of retries on error
+     */
+    this.maxRetriesOnError = opts.maxRetriesOnError;
 
     /**
-     * Stream function arguments
+     * Current amount of retries on error
+     * (Does not have effect if maxRetriesOnError === -1)
+     */
+    this.retriesOnError = 0;
+
+    /**
+     * createStream arguments
      */
     this.args = null;
+    /**
+     * grpc stream
+     */
+    this.stream = null;
+
+    /**
+     * Function to wrap around to handle interval reconnects and error retries
+     */
+    this.createStream = createStream;
+
+    /**
+     * Reference to setTimeout managed by autoReconnectInterval
+     */
+    this.reconnectTimeout = null;
+
+    // For mocks
+    this.setTimeout = (callback, delay) => setTimeout(callback, delay);
+    this.clearTimeout = (reference) => clearTimeout(reference);
 
     this.connect = this.connect.bind(this);
     this.reconnect = this.reconnect.bind(this);
     this.cancel = this.cancel.bind(this);
-    this.destroy = this.destroy.bind(this);
     this.errorHandler = this.errorHandler.bind(this);
     this.dataHandler = this.dataHandler.bind(this);
     this.endHandler = this.endHandler.bind(this);
     this.addListeners = this.addListeners.bind(this);
-
-    // Manages retry attempts when error happens
-    this.retriesOnError = 0;
   }
 
   async connect(...args) {
+    // Memorize current stream args (which can be altered by beforeReconnect logic)
     this.args = args;
-    this.stream = await this.streamFunction(...this.args);
+
+    // Create grpc stream
+    this.stream = await this.createStream(...this.args);
+
+    // Add event listeners
     this.addListeners();
 
-    if (this.reconnectTimeoutDelay > 0) {
-      this.reconnectTimeout = this.setTimeout(
-        this.reconnect,
-        this.reconnectTimeoutDelay,
-      );
+    if (this.autoReconnectInterval > 0) {
+      this.startAutoReconnect();
     }
   }
 
+  startAutoReconnect() {
+    if (this.reconnectTimeout) {
+      throw new Error('Auto reconnect timeout is already running.');
+    }
+    this.reconnectTimeout = this.setTimeout(
+      this.reconnect,
+      this.autoReconnectInterval,
+    );
+  }
+
   reconnect() {
     if (this.reconnectTimeout) {
       this.reconnectTimeout = null;
       this.stream.cancel();
 
-      let newArgs = this.args;
+      let newArgs = this.args || [];
       const updateArgs = (...args) => {
         newArgs = args;
       };
@@ -91,6 +152,10 @@ class ReconnectableStream extends EventEmitter {
   }
 
   /**
+   * Function that adds EventEmitter style listeners
+   * to the stream and also rewires stream cancel function
+   * in order to automatically unsubscribe from events
+   *
    * @private
    */
   addListeners() {
@@ -100,6 +165,10 @@ class ReconnectableStream extends EventEmitter {
 
     const { cancel } = this.stream;
 
+    // Rewire cancel function in order to
+    // unsubscribe from DATA and END events.
+    // We don't unsubscribe from ERROR event because it
+    // handles `CANCELLED_ON_CLIENT` error after calling .cancel() and must stay
     this.stream.cancel = () => {
       this.stream.removeListener(EVENTS.DATA, this.dataHandler);
       this.stream.removeListener(EVENTS.END, this.endHandler);
@@ -107,72 +176,107 @@ class ReconnectableStream extends EventEmitter {
     };
   }
 
+  /**
+   * stream.on('data') handler
+   *
+   * @param data
+   */
   dataHandler(data) {
     this.emit(EVENTS.DATA, data);
   }
 
   /**
+   * stream.on('end') handler
+   *
    * @private
    */
   endHandler() {
-    this.stopReconnectTimeout();
+    this.stopAutoReconnect();
     this.stream = null;
     this.emit(EVENTS.END);
   }
 
   /**
+   * stream.on('error') handler
+   *
    * @private
    * @param e
    */
   errorHandler(e) {
-    this.stream.removeListener(EVENTS.ERROR, this.errorHandler);
+    // In case of cancellation nothing has to happen.
+    // TODO: do we need to propagate GrpcErrorCodes.CANCELLED further?
     if (e.code === GrpcErrorCodes.CANCELLED) {
       return;
     }
-    this.stopReconnectTimeout();
 
-    if (
-      this.maxRetriesOnError === -1 // Infinite retries
-      || this.retriesOnError < this.maxRetriesOnError // Limited retries
-    ) {
-      let newArgs = this.args;
-      const updateArgs = (...args) => {
+    // Retry stream
+    this.retryOnError(e);
+  }
+
+  /**
+   * Manages retry logic
+   *
+   * @param e
+   */
+  retryOnError(e) {
+    // Stop reconnect timeout if there is one
+    this.stopAutoReconnect();
+
+    // Unsubscribe from events in case of the external call
+    this.stream.removeListener(EVENTS.END, this.endHandler);
+    this.stream.removeListener(EVENTS.DATA, this.dataHandler);
+
+    const canRetry = this.maxRetriesOnError === -1 // Infinite retries
+      || this.retriesOnError < this.maxRetriesOnError; // Or less than max limit
+
+    if (canRetry) {
+      // Handle 'beforeReconnect` logic
+      let newArgs = this.args || [];
+      const updateArgsBeforeReconnect = (...args) => {
         newArgs = args;
       };
 
-      this.emit(EVENTS.BEFORE_RECONNECT, updateArgs);
+      // Emit beforeReconnect event in case parent would want to alter arguments
+      this.emit(EVENTS.BEFORE_RECONNECT, updateArgsBeforeReconnect);
 
+      // Wait before reconnecting
       wait(this.retryOnErrorDelay)
-        .then(() => this.connect(...newArgs))
+        .then(() => this.connect(...newArgs))// Reconnect with new args
         .then(() => {
-          this.retriesOnError += 1;
+          this.retriesOnError += 1; // Increase amount of current retries
         })
         .catch((connectError) => {
-          this.emit(EVENTS.ERROR, connectError);
+          this.emit(EVENTS.ERROR, connectError); // Or emit error in case retry has failed
         });
     } else {
+      // Or simply emit error in case retry attempts were exhausted
       this.emit(EVENTS.ERROR, e);
     }
   }
 
   /**
+   * Stops auto reconnect timeout
+   *
    * @private
    */
-  stopReconnectTimeout() {
+  stopAutoReconnect() {
     if (this.reconnectTimeout) {
       this.clearTimeout(this.reconnectTimeout);
       this.reconnectTimeout = null;
     }
   }
 
+  /**
+   * Cancels stream.
+   * Returns `stream.cancel()` to handle it from parent if needed.
+   * (grpc-js cancel() is asynchronous and grpc-web is synchronous)
+   *
+   * @returns {*}
+   */
   cancel() {
-    this.stopReconnectTimeout();
+    this.stopAutoReconnect();
     return this.stream.cancel();
   }
-
-  destroy(e) {
-    this.stream.destroy(e);
-  }
 }
 
 ReconnectableStream.EVENTS = EVENTS;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dashevo/dapi-client",
-  "version": "0.23.0",
+  "version": "0.23.1",
   "description": "Client library used to access Dash DAPI endpoints",
   "main": "lib/index.js",
   "contributors": [
@@ -26,16 +26,15 @@
     }
   ],
   "dependencies": {
-    "@dashevo/dapi-grpc": "0.23.0",
-    "@dashevo/dash-spv": "0.23.0",
+    "@dashevo/dapi-grpc": "0.23.1",
+    "@dashevo/dash-spv": "0.23.1",
     "@dashevo/dashcore-lib": "~0.19.44",
-    "@dashevo/dpp": "0.23.0",
-    "@dashevo/grpc-common": "0.23.0",
-    "@grpc/grpc-js": "^1.3.7",
-    "axios": "^0.27.2",
+    "@dashevo/dpp": "0.23.1",
+    "@dashevo/grpc-common": "0.23.1",
     "bs58": "^4.0.1",
     "cbor": "^8.0.0",
     "lodash.sample": "^4.2.1",
+    "node-fetch": "^2.6.7",
     "node-inspect-extracted": "^1.0.8",
     "wasm-x11-hash": "~0.0.2"
   },