hypercore 10.34.0 → 10.35.0

package/README.md

@@ -74,7 +74,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
   encryptionKey: k, // optionally pass an encryption key to enable block encryption
   onwait: () => {}, // hook that is called if gets are waiting for download
   timeout: 0, // wait at max some milliseconds (0 means no timeout)
-  writable: true // disable appends and truncates
+  writable: true, // disable appends and truncates
+  inflightRange: null // Advanced option. Set to [minInflight, maxInflight] to change the min and max inflight blocks per peer when downloading.
 }
 ```
 

package/index.js

@@ -60,6 +60,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
     this.replicator = null
+    this.inflightRange = opts.inflightRange || null
     this.encryption = null
     this.extensions = new Map()
     this.cache = createCache(opts.cache)
@@ -404,6 +405,7 @@ module.exports = class Hypercore extends EventEmitter {
       eagerUpgrade: true,
       notDownloadingLinger: opts.notDownloadingLinger,
       allowFork: opts.allowFork !== false,
+      inflightRange: this.inflightRange,
       onpeerupdate: this._onpeerupdate.bind(this),
       onupload: this._onupload.bind(this),
       oninvalid: this._oninvalid.bind(this)

package/lib/replicator.js

@@ -1,3 +1,27 @@
+/* DEV DOCS
+  Every hypercore has one Replicator object managing its connections to other peers.
+  There is one Peer object per peer connected to the Hypercore.
+  Hypercores do not know about other hypercores, so when a peer is connected to multiple cores, there exists one Peer object per core.
+
+  Hypercore indicates block should be downloaded through methods like Replicator.addRange or Replicator.addBlock
+  Hypercore calls Replicator.updateActivity every time a hypercore session opens/closes
+  Replicator.updateActivity ensures the Hypercore is downloading blocks as expected
+  Replicator keeps track of:
+    - Which blocks need to be downloaded (Replicator._blocks)
+    - Which blocks currently have inflight requests (Replicator._inflight)
+
+  Blocks are requested from remote peers by Peer objects. The flow is:
+    - The replicator's updatePeer method gets called
+    - The replicator detects whether the Peer can accept more requests (for example by checking if it's maxed out on inflight blocks)
+    - The replicator then tells the Peer what to request (e.g. Peer_requestRange or Peer._requestBlock methods)
+
+  The Peer object is responsible for tracking
+    - Which blocks does the Peer have available (tracked in remoteBitfield)
+    - Which blocks are you actively looking for from this peer (tracked in missingBlocks)
+    - How many blocks are currently inflight (tracked in inflight)
+  The Peer uses this information to decide which blocks to request form the peer in response to _requestRange requests and the like.
+*/
+
 const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
@@ -268,7 +292,7 @@ class BlockTracker {
 }
 
 class Peer {
-  constructor (replicator, protomux, channel, session) {
+  constructor (replicator, protomux, channel, session, inflightRange) {
     this.tracer = createTracer(this, { parent: replicator.core.tracer })
     this.core = replicator.core
     this.replicator = replicator
@@ -276,6 +300,7 @@ class Peer {
     this.protomux = protomux
     this.remotePublicKey = this.stream.remotePublicKey
     this.remoteSupportsSeeks = false
+    this.inflightRange = inflightRange
 
     this.paused = false
 
@@ -299,7 +324,6 @@ class Peer {
     this.receiverBusy = false
 
     this.inflight = 0
-    this.inflightRange = DEFAULT_MAX_INFLIGHT
     this.dataProcessing = 0
 
     this.canUpgrade = true
@@ -1087,7 +1111,7 @@ class Peer {
       return false
     }
 
-    const end = Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength)
+    const end = Math.min(this.core.tree.length, Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength))
     if (end <= r.start || fork !== this.remoteFork) return false
 
     const len = end - r.start
@@ -1215,10 +1239,13 @@ class Peer {
 }
 
 module.exports = class Replicator {
+  static Peer = Peer // hack to be able to access Peer from outside this module
+
   constructor (core, key, {
     notDownloadingLinger = NOT_DOWNLOADING_SLACK,
     eagerUpgrade = true,
     allowFork = true,
+    inflightRange = null,
     onpeerupdate = noop,
     onupload = noop,
     oninvalid = noop
@@ -1239,6 +1266,8 @@ module.exports = class Replicator {
     this.downloading = false
     this.activeSessions = 0
 
+    this.inflightRange = inflightRange || DEFAULT_MAX_INFLIGHT
+
     this._attached = new Set()
     this._inflight = new InflightTracker()
     this._blocks = new BlockTracker()
@@ -2088,7 +2117,7 @@ module.exports = class Replicator {
 
     if (channel === null) return onnochannel()
 
-    const peer = new Peer(replicator, protomux, channel, session)
+    const peer = new Peer(replicator, protomux, channel, session, this.inflightRange)
     const stream = protomux.stream
 
     peer.channel.open({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.34.0",
+  "version": "10.35.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {