hypercore 10.0.0-alpha.0 → 10.0.0-alpha.12

package/lib/protocol.js

@@ -1,4 +1,5 @@
 const { uint, from: fromEncoding } = require('compact-encoding')
+const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const codecs = require('codecs')
 
@@ -125,6 +126,7 @@ class Peer {
     this.resend = false
     this.state = state
     this.extensions = new Map()
+    this.destroyed = false
 
     this._destroyer = this._safeDestroy.bind(this)
   }
@@ -229,6 +231,7 @@ class Peer {
   }
 
   destroy (err) {
+    this.destroyed = true
     return this.protocol.unregisterPeer(this, err)
   }
 }
@@ -283,20 +286,20 @@ module.exports = class Protocol {
 
   registerPeer (key, discoveryKey, handlers = {}, state = null) {
     const peer = new Peer(this, this._localAliases++, key, discoveryKey, handlers, state)
-    this._peers.set(discoveryKey.toString('hex'), peer)
+    this._peers.set(b4a.toString(discoveryKey, 'hex'), peer)
     this._announceCore(peer.alias, key, discoveryKey)
     return peer
   }
 
   unregisterPeer (peer, err) {
-    this._peers.delete(peer.discoveryKey.toString('hex'))
+    this._peers.delete(b4a.toString(peer.discoveryKey, 'hex'))
 
     if (peer.remoteAlias > -1) {
       this._remoteAliases[peer.remoteAlias] = null
       peer.remoteAlias = -1
     }
 
-    peer.handlers.onunregister(this, err)
+    peer.handlers.onunregister(peer, err)
 
     if (err) this.noiseStream.destroy(err)
   }
@@ -372,7 +375,7 @@ module.exports = class Protocol {
     this.send(2, messages.core, -1, {
       alias: alias,
       discoveryKey: discoveryKey,
-      capability: Buffer.alloc(32) // TODO
+      capability: b4a.alloc(32) // TODO
     })
   }
 
@@ -441,7 +444,7 @@ module.exports = class Protocol {
   }
 
   _oncore (m) {
-    const hex = m.discoveryKey.toString('hex')
+    const hex = b4a.toString(m.discoveryKey, 'hex')
     const peer = this._peers.get(hex)
 
     // allow one alloc
@@ -477,7 +480,7 @@ module.exports = class Protocol {
   }
 
   _onunknowncore (m) {
-    const peer = this._peers.get(m.discoveryKey.toString('hex'))
+    const peer = this._peers.get(b4a.toString(m.discoveryKey, 'hex'))
     if (!peer) return
 
     peer.resend = true

package/lib/replicator.js

@@ -1,6 +1,7 @@
 const Protocol = require('./protocol')
 const RemoteBitfield = require('./remote-bitfield')
 const RandomIterator = require('random-array-iterator')
+const b4a = require('b4a')
 
 const PKG = require('../package.json')
 const USER_AGENT = PKG.name + '/' + PKG.version + '@nodejs'
@@ -25,11 +26,12 @@ class InvertedPromise {
 }
 
 class Request {
-  constructor (index, seek) {
+  constructor (index, seek, nodes) {
     this.peer = null
     this.index = index
     this.seek = seek
     this.value = seek === 0
+    this.nodes = nodes
     this.promises = []
   }
 
@@ -119,9 +121,10 @@ class Seek {
 }
 
 class Range {
-  constructor (start, end, linear) {
+  constructor (start, end, filter, linear) {
     this.start = start
     this.end = end
+    this.filter = filter
     this.linear = !!linear
     this.promise = null
     this.done = false
@@ -144,7 +147,7 @@ class Range {
     }
 
     for (; this._start < this.end; this._start++) {
-      if (!bitfield.get(this._start)) return false
+      if (this.filter(this._start) && !bitfield.get(this._start)) return false
     }
 
     return true
@@ -278,7 +281,9 @@ class RequestPool {
 
   _updateSeek (peer, seek) {
     if (seek.request) return false
-    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes)
+    // We have to snapshot the nodes here now, due to the caching of the request...
+    const nodes = log2(seek.seeker.end - seek.seeker.start)
+    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes, nodes)
     return seek.request !== null
   }
 
@@ -296,10 +301,13 @@ class RequestPool {
     const end = range.end === -1 ? peer.state.length : range.end
     if (end <= range._start) return false
 
-    if (range.linear) return !!this._requestRange(peer, range._start, end, 0)
+    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0, range.filter)
 
     const r = range._start + Math.floor(Math.random() * (end - range._start))
-    return !!(this._requestRange(peer, r, end, 0) || this._requestRange(peer, range._start, r, 0))
+    return !!(
+      this._requestRange(peer, r, end, 0, 0, range.filter) ||
+      this._requestRange(peer, range._start, r, 0, 0, range.filter)
+    )
   }
 
   _updateUpgrade (peer) {
@@ -328,7 +336,7 @@ class RequestPool {
     this.pendingUpgrade = null
   }
 
-  _requestRange (peer, start, end, seek) {
+  _requestRange (peer, start, end, seek, nodes, filter = tautology) {
     const remote = peer.state.bitfield
     const local = this.core.bitfield
 
@@ -336,12 +344,12 @@ class RequestPool {
     if (end === -1) end = peer.state.length
 
     for (let i = start; i < end; i++) {
-      if (!remote.get(i) || local.get(i)) continue
+      if (!filter(i) || !remote.get(i) || local.get(i)) continue
       // TODO: if this was a NO_VALUE request, retry if no blocks can be found elsewhere
       if (this.requests.has(i)) continue
 
       // TODO: if seeking and i >= core.length, let that takes precendance in the upgrade req
-      const req = new Request(i, i < this.core.tree.length ? seek : 0)
+      const req = new Request(i, i < this.core.tree.length ? seek : 0, nodes)
       this.requests.set(i, req)
       this.send(peer, req)
       return req
@@ -390,16 +398,28 @@ class RequestPool {
 
     if (data.block.index < this.core.tree.length || this.core.truncating > 0) {
       try {
-        data.block.nodes = await this.core.tree.nodes(data.block.index * 2)
+        data.block.nodes = Math.max(req.nodes, await this.core.tree.nodes(data.block.index * 2))
       } catch (err) {
         console.error('TODO handle me:', err.stack)
       }
     }
 
+    if (peer.destroyed) {
+      req.peer = null
+      this.pending.push(req)
+      if (upgrading) {
+        this.upgrading.resolve()
+        this.upgrading = null
+      }
+      this.replicator.updateAll()
+      return
+    }
+
     if (fork !== this.core.tree.fork || paused(peer, this.core.tree.fork) || this.core.truncating > 0) {
       if (peer.state.inflight > 0) peer.state.inflight--
       if (req.promises.length) { // someone is eagerly waiting for this request
         req.peer = null // resend on some other peer
+        this.pending.push(req)
       } else { // otherwise delete the request
         this.requests.delete(req.index)
       }
@@ -594,7 +614,7 @@ class RequestPool {
       return e.createPromise()
     }
 
-    const r = new Request(index, 0)
+    const r = new Request(index, 0, 0)
 
     this.requests.set(index, r)
     this.pending.push(r)
@@ -662,8 +682,18 @@ module.exports = class Replicator {
     return promise
   }
 
-  static createRange (start, end, linear) {
-    return new Range(start, end, linear)
+  static createRange (start, end, filter, linear) {
+    // createRange(start, end)
+    if (filter === undefined) {
+      filter = tautology
+
+    // createRange(start, end, linear)
+    } else if (typeof filter === 'boolean') {
+      linear = filter
+      filter = tautology
+    }
+
+    return new Range(start, end, filter, linear)
   }
 
   addRange (range) {
@@ -750,8 +780,8 @@ module.exports = class Replicator {
 
   onbitfield ({ start, bitfield }, peer) {
     if (bitfield.length < 1024) {
-      const buf = Buffer.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
-      const bigger = Buffer.concat([buf, Buffer.alloc(4096 - buf.length)])
+      const buf = b4a.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
+      const bigger = b4a.concat([buf, b4a.alloc(4096 - buf.length)])
       bitfield = new Uint32Array(bigger.buffer, bigger.byteOffset, 1024)
     }
     peer.state.bitfield.pages.set(start, bitfield)
@@ -810,3 +840,18 @@ function pages (core) {
 }
 
 function noop () {}
+
+function tautology () {
+  return true
+}
+
+function log2 (n) {
+  let res = 1
+
+  while (n > 2) {
+    n /= 2
+    res++
+  }
+
+  return res
+}

package/lib/streams.js

@@ -0,0 +1,39 @@
+const { Readable } = require('streamx')
+
+class ReadStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this.core = core
+    this.start = opts.start || 0
+    this.end = typeof opts.end === 'number' ? opts.end : -1
+    this.snapshot = !opts.live && opts.snapshot !== false
+    this.live = !!opts.live
+  }
+
+  _open (cb) {
+    this._openP().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readP().then(cb, cb)
+  }
+
+  async _openP () {
+    if (this.end === -1) await this.core.update()
+    else await this.core.ready()
+    if (this.snapshot && this.end === -1) this.end = this.core.length
+  }
+
+  async _readP () {
+    const end = this.live ? -1 : (this.end === -1 ? this.core.length : this.end)
+    if (end >= 0 && this.start >= end) {
+      this.push(null)
+      return
+    }
+
+    this.push(await this.core.get(this.start++))
+  }
+}
+
+exports.ReadStream = ReadStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.0",
+  "version": "10.0.0-alpha.12",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -29,17 +29,19 @@
   "homepage": "https://github.com/hypercore-protocol/hypercore#readme",
   "dependencies": {
     "@hyperswarm/secret-stream": "^5.0.0",
+    "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
     "codecs": "^2.2.0",
-    "compact-encoding": "^2.0.0",
+    "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",
-    "flat-tree": "^1.7.0",
-    "hypercore-crypto": "^2.1.1",
+    "flat-tree": "^1.9.0",
+    "hypercore-crypto": "^3.0.0",
     "is-options": "^1.0.1",
     "random-access-file": "^2.1.4",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
-    "uint64le": "^1.0.0"
+    "sodium-universal": "^3.0.4",
+    "xache": "^1.0.0"
   },
   "devDependencies": {
     "brittle": "^1.4.1",

package/test/basic.js

@@ -56,19 +56,18 @@ test('close multiple', async function (t) {
   const core = await create()
   await core.append('hello world')
 
-  const expected = ['close event', 'close 1', 'close 2', 'close 3']
+  const ev = t.test('events')
 
-  core.on('close', () => done('close event'))
-  core.close().then(() => done('close 1'))
-  core.close().then(() => done('close 2'))
-  core.close().then(() => done('close 3'))
+  ev.plan(4)
 
-  await core.close()
-  t.is(expected.length, 0, 'all event passed')
+  let i = 0
 
-  function done (event) {
-    t.is(event, expected.shift())
-  }
+  core.on('close', () => ev.is(i++, 0, 'on close'))
+  core.close().then(() => ev.is(i++, 1, 'first close'))
+  core.close().then(() => ev.is(i++, 2, 'second close'))
+  core.close().then(() => ev.is(i++, 3, 'third close'))
+
+  await ev
 })
 
 test('storage options', async function (t) {
@@ -77,3 +76,15 @@ test('storage options', async function (t) {
   t.alike(await core.get(0), Buffer.from('hello'))
   t.end()
 })
+
+test(
+  'allow publicKeys with different byteLength that 32, if opts.crypto were passed',
+  function (t) {
+    const key = Buffer.alloc(33).fill('a')
+
+    const core = new Hypercore(ram, key, { crypto: {} })
+
+    t.is(core.key, key)
+    t.pass('creating a core with more than 32 byteLength key did not throw')
+  }
+)

package/test/encryption.js

@@ -0,0 +1,123 @@
+const test = require('brittle')
+const RAM = require('random-access-memory')
+const Hypercore = require('..')
+const { create, replicate } = require('./helpers')
+
+const encryptionKey = Buffer.alloc(32, 'hello world')
+
+test('encrypted append and get', async function (t) {
+  const a = await create({ encryptionKey })
+
+  t.alike(a.encryptionKey, encryptionKey)
+
+  await a.append(['hello'])
+
+  t.is(a.byteLength, 5)
+  t.is(a.core.tree.byteLength, 5 + a.padding)
+
+  const unencrypted = await a.get(0)
+  t.alike(unencrypted, Buffer.from('hello'))
+
+  const encrypted = await a.core.blocks.get(0)
+  t.absent(encrypted.includes('hello'))
+})
+
+test('encrypted seek', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['hello', 'world', '!'])
+
+  t.alike(await a.seek(0), [0, 0])
+  t.alike(await a.seek(4), [0, 4])
+  t.alike(await a.seek(5), [1, 0])
+  t.alike(await a.seek(6), [1, 1])
+  t.alike(await a.seek(6), [1, 1])
+  t.alike(await a.seek(9), [1, 4])
+  t.alike(await a.seek(10), [2, 0])
+  t.alike(await a.seek(11), [3, 0])
+})
+
+test('encrypted replication', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  t.test('with encryption key', async function (t) {
+    const b = await create(a.key, { encryptionKey })
+
+    replicate(a, b, t)
+
+    await t.test('through direct download', async function (t) {
+      const r = b.download({ start: 0, end: a.length })
+      await r.downloaded()
+
+      for (let i = 0; i < 5; i++) {
+        t.alike(await b.get(i), await a.get(i))
+      }
+    })
+
+    await t.test('through indirect download', async function (t) {
+      await a.append(['f', 'g', 'h', 'i', 'j'])
+
+      for (let i = 5; i < 10; i++) {
+        t.alike(await b.get(i), await a.get(i))
+      }
+    })
+  })
+
+  t.test('without encryption key', async function (t) {
+    const b = await create(a.key)
+
+    replicate(a, b, t)
+
+    await t.test('through direct download', async function (t) {
+      const r = b.download({ start: 0, end: a.length })
+      await r.downloaded()
+
+      for (let i = 0; i < 5; i++) {
+        t.alike(await b.get(i), await a.core.blocks.get(i))
+      }
+    })
+
+    await t.test('through indirect download', async function (t) {
+      await a.append(['f', 'g', 'h', 'i', 'j'])
+
+      for (let i = 5; i < 10; i++) {
+        t.alike(await b.get(i), await a.core.blocks.get(i))
+      }
+    })
+  })
+})
+
+test('encrypted session', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['hello'])
+
+  const s = a.session()
+
+  t.alike(a.encryptionKey, s.encryptionKey)
+  t.alike(await s.get(0), Buffer.from('hello'))
+
+  await s.append(['world'])
+
+  const unencrypted = await s.get(1)
+  t.alike(unencrypted, Buffer.from('world'))
+  t.alike(await a.get(1), unencrypted)
+
+  const encrypted = await s.core.blocks.get(1)
+  t.absent(encrypted.includes('world'))
+  t.alike(await a.core.blocks.get(1), encrypted)
+})
+
+test('encrypted session before ready core', async function (t) {
+  const a = new Hypercore(RAM, { encryptionKey })
+  const s = a.session()
+
+  await a.ready()
+
+  t.alike(a.encryptionKey, s.encryptionKey)
+
+  await a.append(['hello'])
+  t.alike(await s.get(0), Buffer.from('hello'))
+})

package/test/extension.js

@@ -1,5 +1,5 @@
 const test = require('brittle')
-const { create, replicate } = require('./helpers')
+const { create, replicate, eventFlush } = require('./helpers')
 
 test('basic extension', async function (t) {
   const messages = ['world', 'hello']
@@ -18,15 +18,15 @@ test('basic extension', async function (t) {
     encoding: 'utf-8'
   })
 
-  replicate(a, b)
+  replicate(a, b, t)
 
-  await new Promise(resolve => setImmediate(resolve))
+  await eventFlush()
   t.is(b.peers.length, 1)
 
   bExt.send('hello', b.peers[0])
   bExt.send('world', b.peers[0])
 
-  await new Promise(resolve => setImmediate(resolve))
+  await eventFlush()
   t.absent(messages.length)
 
   t.end()
@@ -38,7 +38,7 @@ test('two extensions', async function (t) {
   const a = await create()
   const b = await create(a.key)
 
-  replicate(a, b)
+  replicate(a, b, t)
 
   b.registerExtension('test-extension-1', {
     encoding: 'utf-8'
@@ -47,12 +47,12 @@ test('two extensions', async function (t) {
     encoding: 'utf-8'
   })
 
-  await new Promise(resolve => setImmediate(resolve))
+  await eventFlush()
   t.is(b.peers.length, 1)
 
   bExt2.send('world', b.peers[0])
 
-  await new Promise(resolve => setImmediate(resolve))
+  await eventFlush()
 
   a.registerExtension('test-extension-2', {
     encoding: 'utf-8',
@@ -64,7 +64,7 @@ test('two extensions', async function (t) {
 
   bExt2.send('hello', b.peers[0])
 
-  await new Promise(resolve => setImmediate(resolve))
+  await eventFlush()
   t.is(messages.length, 1) // First message gets ignored
 
   t.end()

package/test/helpers/index.js

@@ -8,12 +8,16 @@ module.exports = {
     return core
   },
 
-  replicate (a, b) {
+  replicate (a, b, t) {
     const s1 = a.replicate(true)
     const s2 = b.replicate(false)
-    s1.on('error', err => console.log('STREAM ERROR:', err))
-    s2.on('error', err => console.log('STREAM ERROR:', err))
+    s1.on('error', err => t.comment(`STREAM ERROR: ${err}`))
+    s2.on('error', err => t.comment(`STREAM ERROR: ${err}`))
     s1.pipe(s2).pipe(s1)
     return [s1, s2]
+  },
+
+  async eventFlush () {
+    await new Promise(resolve => setImmediate(resolve))
   }
 }