hypercore 10.0.0-alpha.1 → 10.0.0-alpha.5

package/.github/workflows/test-node.yml

@@ -1,5 +1,4 @@
 name: Build Status
-
 on:
   push:
     branches:
@@ -11,7 +10,7 @@ jobs:
   build:
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [lts/*]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

package/index.js

@@ -3,6 +3,7 @@ const raf = require('random-access-file')
 const isOptions = require('is-options')
 const hypercoreCrypto = require('hypercore-crypto')
 const c = require('compact-encoding')
+const Xache = require('xache')
 const NoiseSecretStream = require('@hyperswarm/secret-stream')
 const codecs = require('codecs')
 
@@ -44,6 +45,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.core = null
     this.replicator = null
     this.extensions = opts.extensions || new Extensions()
+    this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
     this.key = key || null
@@ -89,8 +91,9 @@ module.exports = class Hypercore extends EventEmitter {
     const directory = storage
     const toLock = opts.lock || 'oplog'
     return function createFile (name) {
-      const lock = name === toLock ? fsctl.lock : null
-      const sparse = name !== toLock ? fsctl.sparse : null
+      const locked = name === toLock || name.endsWith('/' + toLock)
+      const lock = locked ? fsctl.lock : null
+      const sparse = locked ? null : null // fsctl.sparse, disable sparse on windows - seems to fail for some people. TODO: investigate
       return raf(name, { directory, lock, sparse })
     }
   }
@@ -280,8 +283,13 @@ module.exports = class Hypercore extends EventEmitter {
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
       for (let i = 0; i < this.sessions.length; i++) {
-        if ((status & 0b10) !== 0) this.sessions[i].emit('truncate', this.core.tree.fork)
-        if ((status & 0b01) !== 0) this.sessions[i].emit('append')
+        if ((status & 0b10) !== 0) {
+          if (this.cache) this.cache.clear()
+          this.sessions[i].emit('truncate', this.core.tree.fork)
+        }
+        if ((status & 0b01) !== 0) {
+          this.sessions[i].emit('append')
+        }
       }
 
       this.replicator.broadcastInfo()
@@ -345,6 +353,15 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
+    const c = this.cache && this.cache.get(index)
+    if (c) return c
+    const fork = this.core.tree.fork
+    const b = await this._get(index, opts)
+    if (this.cache && fork === this.core.tree.fork && b) this.cache.set(index, b)
+    return b
+  }
+
+  async _get (index, opts) {
     const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
 
     if (this.core.bitfield.get(index)) return decode(encoding, await this.core.blocks.get(index))

package/lib/protocol.js

@@ -125,6 +125,7 @@ class Peer {
     this.resend = false
     this.state = state
     this.extensions = new Map()
+    this.destroyed = false
 
     this._destroyer = this._safeDestroy.bind(this)
   }
@@ -229,6 +230,7 @@ class Peer {
   }
 
   destroy (err) {
+    this.destroyed = true
     return this.protocol.unregisterPeer(this, err)
   }
 }
@@ -296,7 +298,7 @@ module.exports = class Protocol {
       peer.remoteAlias = -1
     }
 
-    peer.handlers.onunregister(this, err)
+    peer.handlers.onunregister(peer, err)
 
     if (err) this.noiseStream.destroy(err)
   }

package/lib/replicator.js

@@ -25,11 +25,12 @@ class InvertedPromise {
 }
 
 class Request {
-  constructor (index, seek) {
+  constructor (index, seek, nodes) {
     this.peer = null
     this.index = index
     this.seek = seek
     this.value = seek === 0
+    this.nodes = nodes
     this.promises = []
   }
 
@@ -278,7 +279,9 @@ class RequestPool {
 
   _updateSeek (peer, seek) {
     if (seek.request) return false
-    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes)
+    // We have to snapshot the nodes here now, due to the caching of the request...
+    const nodes = log2(seek.seeker.end - seek.seeker.start)
+    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes, nodes)
     return seek.request !== null
   }
 
@@ -296,10 +299,10 @@ class RequestPool {
     const end = range.end === -1 ? peer.state.length : range.end
     if (end <= range._start) return false
 
-    if (range.linear) return !!this._requestRange(peer, range._start, end, 0)
+    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0)
 
     const r = range._start + Math.floor(Math.random() * (end - range._start))
-    return !!(this._requestRange(peer, r, end, 0) || this._requestRange(peer, range._start, r, 0))
+    return !!(this._requestRange(peer, r, end, 0, 0) || this._requestRange(peer, range._start, r, 0, 0))
   }
 
   _updateUpgrade (peer) {
@@ -328,7 +331,7 @@ class RequestPool {
     this.pendingUpgrade = null
   }
 
-  _requestRange (peer, start, end, seek) {
+  _requestRange (peer, start, end, seek, nodes) {
     const remote = peer.state.bitfield
     const local = this.core.bitfield
 
@@ -341,7 +344,7 @@ class RequestPool {
       if (this.requests.has(i)) continue
 
       // TODO: if seeking and i >= core.length, let that takes precendance in the upgrade req
-      const req = new Request(i, i < this.core.tree.length ? seek : 0)
+      const req = new Request(i, i < this.core.tree.length ? seek : 0, nodes)
       this.requests.set(i, req)
       this.send(peer, req)
       return req
@@ -390,16 +393,28 @@ class RequestPool {
 
     if (data.block.index < this.core.tree.length || this.core.truncating > 0) {
       try {
-        data.block.nodes = await this.core.tree.nodes(data.block.index * 2)
+        data.block.nodes = Math.max(req.nodes, await this.core.tree.nodes(data.block.index * 2))
       } catch (err) {
         console.error('TODO handle me:', err.stack)
       }
     }
 
+    if (peer.destroyed) {
+      req.peer = null
+      this.pending.push(req)
+      if (upgrading) {
+        this.upgrading.resolve()
+        this.upgrading = null
+      }
+      this.replicator.updateAll()
+      return
+    }
+
     if (fork !== this.core.tree.fork || paused(peer, this.core.tree.fork) || this.core.truncating > 0) {
       if (peer.state.inflight > 0) peer.state.inflight--
       if (req.promises.length) { // someone is eagerly waiting for this request
         req.peer = null // resend on some other peer
+        this.pending.push(req)
       } else { // otherwise delete the request
         this.requests.delete(req.index)
       }
@@ -594,7 +609,7 @@ class RequestPool {
       return e.createPromise()
     }
 
-    const r = new Request(index, 0)
+    const r = new Request(index, 0, 0)
 
     this.requests.set(index, r)
     this.pending.push(r)
@@ -810,3 +825,14 @@ function pages (core) {
 }
 
 function noop () {}
+
+function log2 (n) {
+  let res = 1
+
+  while (n > 2) {
+    n /= 2
+    res++
+  }
+
+  return res
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.1",
+  "version": "10.0.0-alpha.5",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -39,7 +39,8 @@
     "random-access-file": "^2.1.4",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
-    "uint64le": "^1.0.0"
+    "uint64le": "^1.0.0",
+    "xache": "^1.0.0"
   },
   "devDependencies": {
     "brittle": "^1.4.1",

package/test/replicate.js

@@ -294,3 +294,40 @@ test('multiplexing multiple times over the same stream', async function (t) {
   t.is(b1.length, a1.length, 'same length')
   t.end()
 })
+
+test('destroying a stream and re-replicating works', async function (t) {
+  const core = await create()
+
+  while (core.length < 33) await core.append(Buffer.from('#' + core.length))
+
+  const clone = await create(core.key)
+
+  let s1 = core.replicate(true)
+  let s2 = clone.replicate(false)
+
+  s1.pipe(s2).pipe(s1)
+
+  await s2.opened
+
+  const all = []
+  for (let i = 0; i < 33; i++) {
+    all.push(clone.get(i))
+  }
+
+  clone.once('download', function () {
+    // simulate stream failure in the middle of bulk downloading
+    s1.destroy()
+  })
+
+  await new Promise((resolve) => s1.once('close', resolve))
+
+  // retry
+  s1 = core.replicate(true)
+  s2 = clone.replicate(false)
+
+  s1.pipe(s2).pipe(s1)
+
+  const blocks = await Promise.all(all)
+
+  t.is(blocks.length, 33, 'downloaded 33 blocks')
+})