@dashevo/dapi-client 5.0.3 → 5.1.0-dev.1

package/lib/BlockHeadersProvider/BlockHeadersProvider.js

@@ -1,13 +1,14 @@
 const EventEmitter = require('events');
 const { SpvChain, SPVError } = require('@dashevo/dash-spv');
+const GrpcErrorCodes = require('@dashevo/grpc-common/lib/server/error/GrpcErrorCodes');
 
 const BlockHeadersReader = require('./BlockHeadersReader');
 
 /**
  * @typedef {BlockHeadersProviderOptions} BlockHeadersProviderOptions
  * @property {string} [network='testnet'] network type
- * @property {number} [maxParallelStreams=5] max parallel streams to read historical block headers
- * @property {number} [targetBatchSize=100000] a target batch size per stream
+ * @property {number} [maxParallelStreams=10] max parallel streams to read historical block headers
+ * @property {number} [targetBatchSize=10000] a target batch size per stream
  * @property {number} [fromBlockHeight=1] a target batch size per stream
  * @property {number} [maxRetries=10] max amount of retries per stream connection
  * @property {number} [powTargetSpacing] devnets only: the -powtargetspacing the
@@ -18,7 +19,9 @@ const BlockHeadersReader = require('./BlockHeadersReader');
 const defaultOptions = {
   network: 'testnet',
   maxParallelStreams: 10,
-  targetBatchSize: 50000,
+  // Headers read ahead of the chain wait in memory, up to about
+  // maxParallelStreams * targetBatchSize of them
+  targetBatchSize: 10000,
   fromBlockHeight: 1,
   maxRetries: 10,
 };
@@ -53,19 +56,25 @@ class BlockHeadersProvider extends EventEmitter {
       ...options,
     };
 
-    const { powTargetSpacing, minimumDifficultyBlocks } = this.options;
-    const devnetOptions = powTargetSpacing !== undefined || minimumDifficultyBlocks !== undefined
-      ? { powTargetSpacing, minimumDifficultyBlocks }
-      : undefined;
-    this.spvChain = new SpvChain(this.options.network, 100, devnetOptions);
-
     this.state = STATES.IDLE;
 
+    // How far below the tip continuous sync asks for headers again when new
+    // ones don't connect: DAPI streams only each new tip, so after a reorg or
+    // several blocks at once the parents have to be fetched by height
+    this.continuousLookback = 1;
+
     this.errorHandler = this.errorHandler.bind(this);
     this.headersHandler = this.headersHandler.bind(this);
+    this.chainConnectedHandler = this.chainConnectedHandler.bind(this);
     this.historicalDataObtainedHandler = this.historicalDataObtainedHandler.bind(this);
     this.createHistoricalSyncStream = createHistoricalSyncStream;
     this.createContinuousSyncStream = createContinuousSyncStream;
+
+    const { powTargetSpacing, minimumDifficultyBlocks } = this.options;
+    const devnetOptions = powTargetSpacing !== undefined || minimumDifficultyBlocks !== undefined
+      ? { powTargetSpacing, minimumDifficultyBlocks }
+      : undefined;
+    this.setSpvChain(new SpvChain(this.options.network, 100, devnetOptions));
   }
 
   /**
@@ -80,7 +89,11 @@ class BlockHeadersProvider extends EventEmitter {
    * @param {SpvChain} spvChain instance
    */
   setSpvChain(spvChain) {
+    if (this.spvChain) {
+      this.spvChain.removeListener(SpvChain.EVENTS.CONNECTED, this.chainConnectedHandler);
+    }
     this.spvChain = spvChain;
+    this.spvChain.on(SpvChain.EVENTS.CONNECTED, this.chainConnectedHandler);
   }
 
   /**
@@ -96,6 +109,9 @@ class BlockHeadersProvider extends EventEmitter {
           maxParallelStreams: this.options.maxParallelStreams,
           targetBatchSize: this.options.targetBatchSize,
           maxRetries: this.options.maxRetries,
+          getConnectedHeight: () => (this.spvChain.startBlockHeight === null
+            ? -Infinity
+            : this.spvChain.getTipHeight()),
         },
         this.createHistoricalSyncStream,
         this.createContinuousSyncStream,
@@ -159,10 +175,16 @@ class BlockHeadersProvider extends EventEmitter {
     this.blockHeadersReader
       .once(BlockHeadersReader.EVENTS.HISTORICAL_DATA_OBTAINED, this.historicalDataObtainedHandler);
 
-    await this.blockHeadersReader.readHistorical(
-      fromBlockHeight,
-      toBlockHeight,
-    );
+    try {
+      await this.blockHeadersReader.readHistorical(
+        fromBlockHeight,
+        toBlockHeight,
+      );
+    } catch (e) {
+      // init() adds them again on the next attempt
+      this.removeReaderListeners();
+      throw e;
+    }
 
     this.state = STATES.HISTORICAL_SYNC;
     this.emit(EVENTS.HISTORICAL_SYNC_STARTED);
@@ -175,7 +197,18 @@ class BlockHeadersProvider extends EventEmitter {
 
     await this.init();
     this.ensureChainRoot(fromBlockHeight);
-    await this.blockHeadersReader.subscribeToNew(fromBlockHeight);
+    this.continuousLookback = 1;
+    this.continuousGapReported = false;
+    try {
+      await this.blockHeadersReader.subscribeToNew(
+        fromBlockHeight,
+        (retryError) => this.getContinuousRetryHeight(retryError),
+      );
+    } catch (e) {
+      // init() adds them again on the next attempt
+      this.removeReaderListeners();
+      throw e;
+    }
     this.state = STATES.CONTINUOUS_SYNC;
     this.emit(EVENTS.CONTINUOUS_SYNC_STARTED);
   }
@@ -225,22 +258,123 @@ class BlockHeadersProvider extends EventEmitter {
     // BlockHeadersReader might not correctly set startBlockHeight
     // which results in orphan chunks in SPV
 
+    let rejection = null;
+    let gapError = null;
     try {
-      const headersAdded = this.spvChain.addHeaders(headers, headHeight);
-
-      if (headersAdded.length) {
-        // Calculate amount of removed headers in order to properly adjust head height
-        // TODO(spv): move this logic to SpvChain?
-        const difference = headers.length - headersAdded.length;
-        this.emit(EVENTS.CHAIN_UPDATED, headersAdded, headHeight + difference);
+      // Headers that reach the main chain come back through chainConnectedHandler
+      this.spvChain.addHeaders(headers, headHeight);
+
+      if (this.state === STATES.CONTINUOUS_SYNC && this.spvChain.getOrphanChunks().length) {
+        // Their parents were never sent: ask again from further below the tip
+        this.spvChain.dropOrphans();
+        if (this.widenContinuousLookback()) {
+          gapError = new SPVError(`New headers don't connect to the chain even ${this.spvChain.confirmsBeforeFinal + 1} headers below its tip`);
+        }
+        rejection = new SPVError(`New headers don't connect to the chain, requesting them again from height ${this.getContinuousResumeHeight()}`);
       }
     } catch (e) {
-      if (e instanceof SPVError) {
-        reject(e);
-      } else {
+      if (!(e instanceof SPVError)) {
         this.errorHandler(e);
+        return;
+      }
+      rejection = e;
+      if (this.state === STATES.CONTINUOUS_SYNC && this.isAtMaxLookback()) {
+        // Such as a reorg that would replace confirmed headers
+        gapError = e;
       }
     }
+
+    // Retry once, then report, outside the validation above
+    if (rejection) {
+      reject(rejection);
+    }
+    if (gapError) {
+      this.reportContinuousGap(gapError);
+    }
+  }
+
+  /**
+   * @private
+   * @param {BlockHeader[]} headers - the main chain from firstHeight on;
+   *   after a reorg firstHeight is at or below the previous tip
+   * @param {number} firstHeight
+   */
+  chainConnectedHandler(headers, firstHeight) {
+    if (this.blockHeadersReader && typeof this.blockHeadersReader.chainAdvanced === 'function') {
+      this.blockHeadersReader.chainAdvanced();
+    }
+    this.continuousLookback = 1;
+    this.continuousGapReported = false;
+    this.emit(EVENTS.CHAIN_UPDATED, headers, firstHeight);
+  }
+
+  /**
+   * @private
+   * Continuous sync keeps retrying headers it can't add, but reports once,
+   * after looking back as far as it can, that it can't get back on the chain
+   * @param {Error} error
+   */
+  reportContinuousGap(error) {
+    if (!this.continuousGapReported && this.listenerCount(EVENTS.ERROR) > 0) {
+      this.continuousGapReported = true;
+      this.emit(EVENTS.ERROR, error);
+    }
+  }
+
+  /** @private */
+  isAtMaxLookback() {
+    return this.continuousLookback === this.spvChain.confirmsBeforeFinal + 1;
+  }
+
+  /**
+   * @private
+   * Makes continuous sync resubscribe from further below the tip
+   * @return {boolean} whether it already looked back as far as it can
+   */
+  widenContinuousLookback() {
+    const wasAtMax = this.isAtMaxLookback();
+    this.continuousLookback = Math.min(
+      this.continuousLookback * 2,
+      this.spvChain.confirmsBeforeFinal + 1,
+    );
+    return wasAtMax;
+  }
+
+  /**
+   * @private
+   * Height continuous sync resubscribes from after the stream failed
+   * @param {Error} [retryError] - the error that made the stream retry
+   * @return {number}
+   */
+  getContinuousRetryHeight(retryError) {
+    // DAPI has no block at the height asked for, as after a reorg to a
+    // shorter chain with more work; without headers arriving, only this
+    // moves the next request below the new tip
+    if (retryError && retryError.code === GrpcErrorCodes.NOT_FOUND
+      && this.state === STATES.CONTINUOUS_SYNC && this.spvChain.startBlockHeight !== null
+      && this.widenContinuousLookback()) {
+      const gapError = new Error(`DAPI has no block even ${this.spvChain.confirmsBeforeFinal + 1} headers below the chain tip: ${retryError.message}`);
+      // The stream schedules its retry after this returns: report once it
+      // has, so an error listener that throws can't stop the retry
+      setTimeout(() => {
+        if (this.state === STATES.CONTINUOUS_SYNC) {
+          this.reportContinuousGap(gapError);
+        }
+      }, 0);
+    }
+    return this.getContinuousResumeHeight();
+  }
+
+  /**
+   * @private
+   * @return {number}
+   */
+  getContinuousResumeHeight() {
+    if (this.spvChain.startBlockHeight === null) {
+      // No headers yet: wait for the first one where sync was asked to start
+      return this.spvChain.pendingStartBlockHeight || 1;
+    }
+    return Math.max(1, this.spvChain.getTipHeight() - this.continuousLookback + 1);
   }
 
   historicalDataObtainedHandler() {

package/lib/BlockHeadersProvider/BlockHeadersReader.js

@@ -15,6 +15,8 @@ const CLIENT_ALREADY_CLOSED_ERROR_MESSAGE = 'Client already closed - cannot .clo
  * @property {number} [maxParallelStreams]
  * @property {number} [targetBatchSize]
  * @property {number} [maxRetries]
+ * @property {function(): number} [getConnectedHeight] - height up to which
+ *   headers joined the chain; bounds how far historical reads run ahead
  */
 
 class BlockHeadersReader extends EventEmitter {
@@ -32,13 +34,31 @@ class BlockHeadersReader extends EventEmitter {
     this.maxParallelStreams = options.maxParallelStreams;
     this.targetBatchSize = options.targetBatchSize;
     this.maxRetries = options.maxRetries;
+    this.getConnectedHeight = options.getConnectedHeight;
 
     /**
-     * Holds references to the historical streams
+     * Holds references to the running historical streams, in batch order
      * @type {Stream[]}
      */
     this.historicalStreams = [];
 
+    /**
+     * Batches of the current historical read that started and haven't
+     * settled yet, in height order; null when not reading
+     * @type {{
+     *   fromBlockHeight: number, count: number, stream: ?Stream, retry: ?Function, done: boolean
+     * }[]|null}
+     */
+    this.historicalBatches = null;
+
+    /**
+     * Where the current historical read's next batch starts, how large its
+     * batches are, and its last height
+     */
+    this.nextBatchHeight = 0;
+    this.historicalBatchSize = 0;
+    this.historicalToBlockHeight = 0;
+
     /**
      * Holds reference to the continuous sync stream
      * @type {Stream}
@@ -47,13 +67,17 @@ class BlockHeadersReader extends EventEmitter {
   }
 
   /**
-   * Reads historical block heights using multiple streams
+   * Reads historical block headers in batches of about targetBatchSize, with
+   * up to maxParallelStreams batches in flight. A batch starts only within
+   * maxParallelStreams batches of the first one not yet settled: finished and,
+   * with getConnectedHeight, joined to the chain. So headers waiting ahead of
+   * the chain stay bounded.
    * @param {number} fromBlockHeight
    * @param {number} toBlockHeight
    * @returns {Promise<void>}
    */
   async readHistorical(fromBlockHeight, toBlockHeight) {
-    if (this.historicalStreams.length) {
+    if (this.historicalBatches) {
       throw new Error('Historical streams are already running. Please stop them first.');
     }
 
@@ -66,37 +90,173 @@ class BlockHeadersReader extends EventEmitter {
       throw new Error(`Invalid total amount of headers to read: ${totalAmount}`);
     }
 
-    const numStreams = Math.min(
-      Math.max(Math.round(totalAmount / this.targetBatchSize), 1),
-      this.maxParallelStreams,
-    );
+    const batchCount = Math.max(Math.round(totalAmount / this.targetBatchSize), 1);
 
-    const actualBatchSize = Math.ceil(totalAmount / numStreams);
+    // Batches are made as the window reaches them: toBlockHeight comes from
+    // a DAPI node, so the read must not allocate per batch up front
+    this.historicalBatches = [];
+    this.historicalBatchSize = Math.ceil(totalAmount / batchCount);
+    this.historicalToBlockHeight = toBlockHeight;
+    this.nextBatchHeight = fromBlockHeight;
 
-    for (let batchIndex = 0; batchIndex < numStreams; batchIndex += 1) {
-      const startingHeight = (batchIndex * actualBatchSize) + fromBlockHeight;
-      const count = Math.min(actualBatchSize, toBlockHeight - startingHeight + 1);
+    try {
+      await this.startHistoricalBatches();
+    } catch (e) {
+      this.stopReadingHistorical();
+      throw e;
+    }
+  }
+
+  /**
+   * @private
+   * Starts the batches the window allows
+   */
+  async startHistoricalBatches() {
+    const batches = this.historicalBatches;
+    while (batches === this.historicalBatches
+      && this.nextBatchHeight <= this.historicalToBlockHeight
+      && batches.length < this.maxParallelStreams) {
+      const batch = {
+        fromBlockHeight: this.nextBatchHeight,
+        count: Math.min(
+          this.historicalBatchSize,
+          this.historicalToBlockHeight - this.nextBatchHeight + 1,
+        ),
+        stream: null,
+        retry: null,
+        done: false,
+      };
+      batches.push(batch);
+      this.nextBatchHeight += batch.count;
 
-      const subscribeWithRetries = this.createSubscribeToHistoricalBatch(this.maxRetries);
+      const subscribeWithRetries = this.createSubscribeToHistoricalBatch(this.maxRetries, batch);
       // eslint-disable-next-line no-await-in-loop
-      const stream = await subscribeWithRetries(startingHeight, count);
-      this.historicalStreams.push(stream);
+      const stream = await subscribeWithRetries(batch.fromBlockHeight, batch.count);
+      if (batches === this.historicalBatches) {
+        batch.stream = stream;
+        this.updateHistoricalStreams();
+      } else {
+        // Stopped while subscribing
+        this.cancelStream(stream);
+      }
     }
   }
 
-  stopReadingHistorical() {
-    while (this.historicalStreams.length) {
-      const stream = this.historicalStreams.shift();
-      this.cancelStream(stream);
+  /** @private */
+  updateHistoricalStreams() {
+    this.historicalStreams = (this.historicalBatches || [])
+      .filter(({ stream }) => stream)
+      .map(({ stream }) => stream);
+  }
+
+  /**
+   * @private
+   * @param {object} batch
+   * @param {Stream} stream
+   */
+  historicalBatchEnded(batch, stream) {
+    if (!batch) {
+      // A stream read on its own
+      this.emit(EVENTS.HISTORICAL_DATA_OBTAINED);
+      return;
     }
+
+    if (batch.stream !== stream || !this.historicalBatches) {
+      // Ended after a retry replaced it or reading stopped
+      return;
+    }
+
+    /* eslint-disable no-param-reassign */
+    batch.stream = null;
+    batch.done = true;
+    /* eslint-enable no-param-reassign */
+    this.updateHistoricalStreams();
+
+    const batches = this.historicalBatches;
+    if (this.nextBatchHeight > this.historicalToBlockHeight && batches.every(({ done }) => done)) {
+      // Every batch arrived; did those not settled yet all join the chain?
+      const connectedHeight = this.getConnectedHeight ? this.getConnectedHeight() : Infinity;
+      const unconnected = batches.find(({ fromBlockHeight, count }) => (
+        connectedHeight < fromBlockHeight + count - 1));
+      if (unconnected) {
+        this.stopReadingHistorical();
+        this.emit(EVENTS.ERROR, new Error(`Headers from height ${unconnected.fromBlockHeight} don't connect to the chain`));
+        return;
+      }
+      this.historicalBatches = null;
+      this.emit(EVENTS.HISTORICAL_DATA_OBTAINED);
+      return;
+    }
+
+    this.advanceHistoricalWindow();
+  }
+
+  /**
+   * Lets historical reads run further ahead once headers join the chain
+   */
+  chainAdvanced() {
+    if (this.historicalBatches) {
+      this.advanceHistoricalWindow();
+    }
+  }
+
+  /** @private */
+  advanceHistoricalWindow() {
+    const batches = this.historicalBatches;
+    const connectedHeight = this.getConnectedHeight ? this.getConnectedHeight() : Infinity;
+    // Retire the batches that arrived and joined the chain
+    while (batches.length) {
+      const { done, fromBlockHeight, count } = batches[0];
+      if (!done || connectedHeight < fromBlockHeight + count - 1) {
+        break;
+      }
+      batches.shift();
+    }
+
+    const windowFull = batches.length >= this.maxParallelStreams;
+    const inFlight = batches.some(({ done }) => !done);
+    if (windowFull && !inFlight) {
+      // Everything in the window arrived, but the first batch didn't connect
+      const { fromBlockHeight } = batches[0];
+      this.stopReadingHistorical();
+      this.emit(EVENTS.ERROR, new Error(`Headers from height ${fromBlockHeight} don't connect to the chain`));
+      return;
+    }
+
+    this.startHistoricalBatches().catch((e) => {
+      if (batches !== this.historicalBatches) {
+        // A later read replaced this one
+        return;
+      }
+      this.stopReadingHistorical();
+      this.emit(EVENTS.ERROR, e);
+    });
+  }
+
+  stopReadingHistorical() {
+    const streams = this.historicalStreams;
+    // Detach the batches, so retries in flight cancel the streams they open
+    (this.historicalBatches || []).forEach((batch) => {
+      /* eslint-disable no-param-reassign */
+      batch.stream = null;
+      batch.retry = null;
+      /* eslint-enable no-param-reassign */
+    });
+    this.historicalBatches = null;
+    this.historicalStreams = [];
+    streams.forEach((stream) => this.cancelStream(stream));
   }
 
   /**
    * Subscribes to continuously arriving block headers
    * @param {number} fromBlockHeight
+   * @param {function(Error=): number} [getResumeHeight] - height to resubscribe
+   *   from after the stream drops or headers are rejected, given the error that
+   *   caused the retry if there was one; by default the height of the last
+   *   header received
    * @returns {Promise<ReconnectableStream>}
    */
-  async subscribeToNew(fromBlockHeight) {
+  async subscribeToNew(fromBlockHeight, getResumeHeight) {
     if (this.continuousSyncStream) {
       throw new Error('Continuous sync has already been started');
     }
@@ -147,13 +307,14 @@ class BlockHeadersReader extends EventEmitter {
       }
     };
 
-    const beforeReconnectHandler = (updateArguments) => {
+    const beforeReconnectHandler = (updateArguments, retryError) => {
+      const resumeHeight = getResumeHeight ? getResumeHeight(retryError) : lastKnownChainHeight;
       updateArguments({
-        fromBlockHeight: lastKnownChainHeight,
+        fromBlockHeight: resumeHeight,
         count: 0,
       });
 
-      lastKnownChainHeight -= 1;
+      lastKnownChainHeight = resumeHeight - 1;
     };
 
     const endHandler = () => {
@@ -189,9 +350,10 @@ class BlockHeadersReader extends EventEmitter {
    * and handles retry logic
    * @private
    * @param {number} [maxRetries] - maximum amount of retries
+   * @param {object} [batch] - the batch the stream reads, from readHistorical
    * @returns {function(*, *): Promise<Stream>}
    */
-  createSubscribeToHistoricalBatch(maxRetries = 0) {
+  createSubscribeToHistoricalBatch(maxRetries = 0, batch = null) {
     let currentRetries = 0;
 
     /**
@@ -205,28 +367,64 @@ class BlockHeadersReader extends EventEmitter {
 
       const stream = await this.createHistoricalSyncStream(fromBlockHeight, count);
 
+      // Events of a stream its batch no longer reads from (it failed and a
+      // retry replaced it, reading stopped, or it already delivered its batch)
+      // are ignored. The reader detaches a stream before it cancels it, so a
+      // CANCELLED error from a current stream came from the server and is
+      // retried like any other.
+      let finished = false;
+      const isCurrent = () => !finished && (!batch || batch.stream === stream);
+
       const errorHandler = (streamError) => {
-        if (streamError.code === GrpcErrorCodes.CANCELLED) {
-          // TODO: consider reworking with COMMANDS instead
-          // of producing a side effect that alters class state
-          const index = this.historicalStreams.indexOf(stream);
-          if (index >= 0) {
-            this.historicalStreams.splice(index, 1);
-          }
+        if (!isCurrent()) {
+          return;
+        }
+
+        if (headersObtained >= count) {
+          // It already delivered its whole batch
+          this.historicalBatchEnded(batch, stream);
+          return;
+        }
+
+        if (!batch && streamError.code === GrpcErrorCodes.CANCELLED) {
           return;
         }
 
         if (currentRetries < maxRetries) {
           const newFromBlockHeight = fromBlockHeight + headersObtained;
           const newCount = count - headersObtained;
+          currentRetries += 1;
+
+          // Detach the failed stream at once, so its late events can't end
+          // the batch while the retry connects
+          const retry = {};
+          if (batch) {
+            /* eslint-disable no-param-reassign */
+            batch.stream = null;
+            batch.retry = retry;
+            /* eslint-enable no-param-reassign */
+            this.updateHistoricalStreams();
+          }
 
-          // TODO: do not retry in case newCount is zero
           subscribeWithRetries(newFromBlockHeight, newCount)
             .then((newStream) => {
-              const index = this.historicalStreams.indexOf(stream);
-              this.historicalStreams[index] = newStream;
-              currentRetries += 1;
+              if (!batch) {
+                return;
+              }
+              if (batch.retry !== retry) {
+                // Reading stopped in the meantime
+                this.cancelStream(newStream);
+                return;
+              }
+              /* eslint-disable no-param-reassign */
+              batch.retry = null;
+              batch.stream = newStream;
+              /* eslint-enable no-param-reassign */
+              this.updateHistoricalStreams();
             }).catch((e) => {
+              if (batch && batch.retry !== retry) {
+                return;
+              }
               this.stopReadingHistorical();
               this.emit(EVENTS.ERROR, e);
             });
@@ -239,9 +437,16 @@ class BlockHeadersReader extends EventEmitter {
       const dataHandler = (data) => {
         const blockHeaders = data.getBlockHeaders();
 
-        if (blockHeaders) {
-          const headersList = blockHeaders.getHeadersList()
-            .map((header) => new BlockHeader(Buffer.from(header)));
+        if (blockHeaders && isCurrent()) {
+          const rawHeaders = blockHeaders.getHeadersList();
+          if (headersObtained + rawHeaders.length > count) {
+            // Never take more than was asked for: the rest would wait in memory
+            errorHandler(new Error(`Stream sent more than the ${count} headers asked for`));
+            this.cancelStream(stream);
+            return;
+          }
+
+          const headersList = rawHeaders.map((header) => new BlockHeader(Buffer.from(header)));
 
           let rejected = false;
 
@@ -251,10 +456,9 @@ class BlockHeadersReader extends EventEmitter {
            */
           const rejectHeaders = (e) => {
             rejected = true;
-            // Cancel stream and unsubscribe from all data events
-            // because they are going th be re-created in errorHandler
-            this.cancelStream(stream);
+            // Retry first, which detaches the stream, then cancel it
             errorHandler(e);
+            this.cancelStream(stream);
           };
 
           const batchHeadHeight = fromBlockHeight + headersObtained;
@@ -265,15 +469,27 @@ class BlockHeadersReader extends EventEmitter {
 
           if (!rejected) {
             headersObtained += headersList.length;
+
+            // Done without waiting for the server to end the stream: close it
+            // before releasing the batch's slot, so a server that never ends
+            // its streams can't keep more than the window open
+            if (headersObtained === count) {
+              finished = true;
+              this.cancelStream(stream);
+              this.historicalBatchEnded(batch, stream);
+            }
           }
         }
       };
 
       const endHandler = () => {
-        const index = this.historicalStreams.indexOf(stream);
-        this.historicalStreams.splice(index, 1);
-        if (this.historicalStreams.length === 0) {
-          this.emit(EVENTS.HISTORICAL_DATA_OBTAINED);
+        if (!isCurrent()) {
+          return;
+        }
+        if (headersObtained >= count) {
+          this.historicalBatchEnded(batch, stream);
+        } else {
+          errorHandler(new Error(`Stream ended after ${headersObtained} of ${count} headers`));
         }
       };
 
@@ -315,6 +531,10 @@ class BlockHeadersReader extends EventEmitter {
 
   cancelStream(stream) {
     stream.removeAllListeners();
+    // The CANCELLED error a cancelled stream emits must not go unheard
+    if (typeof stream.listenerCount === 'function' && stream.listenerCount('error') === 0) {
+      stream.on('error', () => {});
+    }
     this.cancelStreamTransport(stream);
   }
 }

package/lib/dapiAddressProvider/ListDAPIAddressProvider.js

@@ -1,5 +1,4 @@
 const sample = (arr) => arr[Math.floor(Math.random() * arr.length)];
-const networks = require('@dashevo/dashcore-lib/lib/networks');
 
 class ListDAPIAddressProvider {
   /**
@@ -20,28 +19,7 @@ class ListDAPIAddressProvider {
    * @returns {Promise<DAPIAddress|undefined>}
    */
   async getLiveAddress() {
-    const liveAddresses = this.getLiveAddresses();
-
-    const liveAddress = sample(liveAddresses);
-
-    if (liveAddress === undefined) {
-      return undefined;
-    }
-
-    // This is a temporary fix for a localhost masternode.
-    // On macOS, internal docker IP is used to register masternode, and it's
-    // not really possible to bind to that address, so that workaround is introduced.
-    const network = networks.get(this.options.network);
-    if (network && network.regtestEnabled) {
-      const randomNodeIndex = Math.floor(Math.random() * liveAddresses.length);
-
-      liveAddress.protocol = 'https';
-      liveAddress.host = '127.0.0.1';
-      liveAddress.allowSelfSignedCertificate = true;
-      liveAddress.port = 2443 + randomNodeIndex * 100;
-    }
-
-    return liveAddress;
+    return sample(this.getLiveAddresses());
   }
 
   /**

package/lib/dapiAddressProvider/SimplifiedMasternodeListDAPIAddressProvider.js

@@ -1,3 +1,5 @@
+const networks = require('@dashevo/dashcore-lib/lib/networks');
+
 const DAPIAddress = require('./DAPIAddress');
 
 class SimplifiedMasternodeListDAPIAddressProvider {
@@ -5,9 +7,11 @@ class SimplifiedMasternodeListDAPIAddressProvider {
    * @param {SimplifiedMasternodeListProvider} smlProvider
    * @param {ListDAPIAddressProvider} listDAPIAddressProvider
    * @param {DAPIAddress[]} addressWhiteList
+   * @param {DAPIClientOptions} [options]
    */
-  constructor(smlProvider, listDAPIAddressProvider, addressWhiteList) {
+  constructor(smlProvider, listDAPIAddressProvider, addressWhiteList, options = {}) {
     this.smlProvider = smlProvider;
+    this.options = options;
     this.listDAPIAddressProvider = listDAPIAddressProvider;
     this.addressWhiteStrings = addressWhiteList.map((dapiAddress) => dapiAddress.toString());
   }
@@ -58,6 +62,18 @@ class SimplifiedMasternodeListDAPIAddressProvider {
       ));
     }
 
+    // Local (regtest) masternodes register a docker-internal IP that is not
+    // reachable from the host, while their gateways are published on the host's
+    // loopback at the registered port with a self-signed certificate.
+    const network = networks.get(this.options.network);
+    if (network && network.regtestEnabled) {
+      filteredAddresses.forEach((address) => {
+        address.setHost('127.0.0.1');
+        // eslint-disable-next-line no-param-reassign
+        address.allowSelfSignedCertificate = true;
+      });
+    }
+
     this.listDAPIAddressProvider.setAddresses(filteredAddresses);
 
     return this.listDAPIAddressProvider.getLiveAddress();

package/lib/dapiAddressProvider/createDAPIAddressProviderFromOptions.js

@@ -93,6 +93,7 @@ function createDAPIAddressProviderFromOptions(options) {
       smlProvider,
       listDAPIAddressProvider,
       dapiAddressesWhiteList.map((rawAddress) => new DAPIAddress(rawAddress)),
+      options,
     );
   }
 

package/lib/networkConfigs.js

@@ -59,7 +59,7 @@ module.exports = {
     ],
   },
   local: {
-    dapiAddresses: ['127.0.0.1'],
+    dapiAddresses: ['127.0.0.1:2443:self-signed'],
     network: 'regtest',
   },
   mainnet: {

package/lib/test/mocks/BlockHeadersWithChainLocksStreamMock.js

@@ -21,11 +21,11 @@ class BlockHeadersWithChainLocksStreamMock extends EventEmitter {
     }
   }
 
-  emit(event, data) {
+  emit(event, ...args) {
     if (event === 'error') {
       this.errored = true;
     }
-    super.emit(event, data);
+    super.emit(event, ...args);
   }
 
   /**

package/lib/transport/ReconnectableStream.js

@@ -256,8 +256,9 @@ class ReconnectableStream extends EventEmitter {
         newArgs = args;
       };
 
-      // Emit beforeReconnect event in case parent would want to alter arguments
-      this.emit(EVENTS.BEFORE_RECONNECT, updateArgsBeforeReconnect);
+      // Emit beforeReconnect event in case parent would want to alter arguments,
+      // with the error that caused the retry
+      this.emit(EVENTS.BEFORE_RECONNECT, updateArgsBeforeReconnect, e);
 
       // Wait before reconnecting
       wait(this.retryOnErrorDelay)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dashevo/dapi-client",
-  "version": "5.0.3",
+  "version": "5.1.0-dev.1",
   "description": "Client library used to access Dash DAPI endpoints",
   "main": "lib/index.js",
   "contributors": [
@@ -26,11 +26,11 @@
     }
   ],
   "dependencies": {
-    "@dashevo/dapi-grpc": "5.0.3",
-    "@dashevo/dash-spv": "6.0.3",
+    "@dashevo/dapi-grpc": "5.1.0-dev.1",
+    "@dashevo/dash-spv": "6.1.0-dev.1",
     "@dashevo/dashcore-lib": "~0.26.1",
-    "@dashevo/grpc-common": "5.0.3",
-    "@dashevo/wasm-dpp": "5.0.3",
+    "@dashevo/grpc-common": "5.1.0-dev.1",
+    "@dashevo/wasm-dpp": "5.1.0-dev.1",
     "cbor": "^8.0.0",
     "google-protobuf": "^3.12.2",
     "undici": "^6.28.1",
@@ -72,7 +72,7 @@
     "webpack-cli": "^4.9.1"
   },
   "engines": {
-    "node": ">=18.18"
+    "node": ">=22"
   },
   "files": [
     "docs",