hypercore 10.0.0-alpha.2 → 10.0.0-alpha.23

package/lib/replicator.js

@@ -1,6 +1,7 @@
 const Protocol = require('./protocol')
 const RemoteBitfield = require('./remote-bitfield')
 const RandomIterator = require('random-array-iterator')
+const b4a = require('b4a')
 
 const PKG = require('../package.json')
 const USER_AGENT = PKG.name + '/' + PKG.version + '@nodejs'
@@ -25,11 +26,12 @@ class InvertedPromise {
 }
 
 class Request {
-  constructor (index, seek) {
+  constructor (index, seek, nodes) {
     this.peer = null
     this.index = index
     this.seek = seek
     this.value = seek === 0
+    this.nodes = nodes
     this.promises = []
   }
 
@@ -119,9 +121,10 @@ class Seek {
 }
 
 class Range {
-  constructor (start, end, linear) {
+  constructor (start, end, filter, linear) {
     this.start = start
     this.end = end
+    this.filter = filter
     this.linear = !!linear
     this.promise = null
     this.done = false
@@ -144,7 +147,7 @@ class Range {
     }
 
     for (; this._start < this.end; this._start++) {
-      if (!bitfield.get(this._start)) return false
+      if (this.filter(this._start) && !bitfield.get(this._start)) return false
     }
 
     return true
@@ -278,7 +281,9 @@ class RequestPool {
 
   _updateSeek (peer, seek) {
     if (seek.request) return false
-    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes)
+    // We have to snapshot the nodes here now, due to the caching of the request...
+    const nodes = log2(seek.seeker.end - seek.seeker.start)
+    seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes, nodes)
     return seek.request !== null
   }
 
@@ -296,10 +301,13 @@ class RequestPool {
     const end = range.end === -1 ? peer.state.length : range.end
     if (end <= range._start) return false
 
-    if (range.linear) return !!this._requestRange(peer, range._start, end, 0)
+    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0, range.filter)
 
     const r = range._start + Math.floor(Math.random() * (end - range._start))
-    return !!(this._requestRange(peer, r, end, 0) || this._requestRange(peer, range._start, r, 0))
+    return !!(
+      this._requestRange(peer, r, end, 0, 0, range.filter) ||
+      this._requestRange(peer, range._start, r, 0, 0, range.filter)
+    )
   }
 
   _updateUpgrade (peer) {
@@ -328,7 +336,7 @@ class RequestPool {
     this.pendingUpgrade = null
   }
 
-  _requestRange (peer, start, end, seek) {
+  _requestRange (peer, start, end, seek, nodes, filter = tautology) {
     const remote = peer.state.bitfield
     const local = this.core.bitfield
 
@@ -336,12 +344,12 @@ class RequestPool {
     if (end === -1) end = peer.state.length
 
     for (let i = start; i < end; i++) {
-      if (!remote.get(i) || local.get(i)) continue
+      if (!filter(i) || !remote.get(i) || local.get(i)) continue
       // TODO: if this was a NO_VALUE request, retry if no blocks can be found elsewhere
       if (this.requests.has(i)) continue
 
       // TODO: if seeking and i >= core.length, let that takes precendance in the upgrade req
-      const req = new Request(i, i < this.core.tree.length ? seek : 0)
+      const req = new Request(i, i < this.core.tree.length ? seek : 0, nodes)
       this.requests.set(i, req)
       this.send(peer, req)
       return req
@@ -390,16 +398,28 @@ class RequestPool {
 
     if (data.block.index < this.core.tree.length || this.core.truncating > 0) {
       try {
-        data.block.nodes = await this.core.tree.nodes(data.block.index * 2)
+        data.block.nodes = Math.max(req.nodes, await this.core.tree.nodes(data.block.index * 2))
       } catch (err) {
         console.error('TODO handle me:', err.stack)
       }
     }
 
+    if (peer.destroyed) {
+      req.peer = null
+      this.pending.push(req)
+      if (upgrading) {
+        this.upgrading.resolve()
+        this.upgrading = null
+      }
+      this.replicator.updateAll()
+      return
+    }
+
     if (fork !== this.core.tree.fork || paused(peer, this.core.tree.fork) || this.core.truncating > 0) {
       if (peer.state.inflight > 0) peer.state.inflight--
       if (req.promises.length) { // someone is eagerly waiting for this request
         req.peer = null // resend on some other peer
+        this.pending.push(req)
       } else { // otherwise delete the request
         this.requests.delete(req.index)
       }
@@ -594,7 +614,7 @@ class RequestPool {
       return e.createPromise()
     }
 
-    const r = new Request(index, 0)
+    const r = new Request(index, 0, 0)
 
     this.requests.set(index, r)
     this.pending.push(r)
@@ -613,8 +633,9 @@ module.exports = class Replicator {
     this.onupdate = onupdate
   }
 
-  static createProtocol (noiseStream) {
+  static createProtocol (noiseStream, opts) {
     return new Protocol(noiseStream, {
+      ...opts,
       protocolVersion: 0,
       userAgent: USER_AGENT
     })
@@ -662,8 +683,18 @@ module.exports = class Replicator {
     return promise
   }
 
-  static createRange (start, end, linear) {
-    return new Range(start, end, linear)
+  static createRange (start, end, filter, linear) {
+    // createRange(start, end)
+    if (filter === undefined) {
+      filter = tautology
+
+    // createRange(start, end, linear)
+    } else if (typeof filter === 'boolean') {
+      linear = filter
+      filter = tautology
+    }
+
+    return new Range(start, end, filter, linear)
   }
 
   addRange (range) {
@@ -750,8 +781,8 @@ module.exports = class Replicator {
 
   onbitfield ({ start, bitfield }, peer) {
     if (bitfield.length < 1024) {
-      const buf = Buffer.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
-      const bigger = Buffer.concat([buf, Buffer.alloc(4096 - buf.length)])
+      const buf = b4a.from(bitfield.buffer, bitfield.byteOffset, bitfield.byteLength)
+      const bigger = b4a.concat([buf, b4a.alloc(4096 - buf.length)])
       bitfield = new Uint32Array(bigger.buffer, bigger.byteOffset, 1024)
     }
     peer.state.bitfield.pages.set(start, bitfield)
@@ -810,3 +841,18 @@ function pages (core) {
 }
 
 function noop () {}
+
+function tautology () {
+  return true
+}
+
+function log2 (n) {
+  let res = 1
+
+  while (n > 2) {
+    n /= 2
+    res++
+  }
+
+  return res
+}

package/lib/streams.js

@@ -0,0 +1,56 @@
+const { Writable, Readable } = require('streamx')
+
+class ReadStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this.core = core
+    this.start = opts.start || 0
+    this.end = typeof opts.end === 'number' ? opts.end : -1
+    this.snapshot = !opts.live && opts.snapshot !== false
+    this.live = !!opts.live
+  }
+
+  _open (cb) {
+    this._openP().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readP().then(cb, cb)
+  }
+
+  async _openP () {
+    if (this.end === -1) await this.core.update()
+    else await this.core.ready()
+    if (this.snapshot && this.end === -1) this.end = this.core.length
+  }
+
+  async _readP () {
+    const end = this.live ? -1 : (this.end === -1 ? this.core.length : this.end)
+    if (end >= 0 && this.start >= end) {
+      this.push(null)
+      return
+    }
+
+    this.push(await this.core.get(this.start++))
+  }
+}
+
+exports.ReadStream = ReadStream
+
+class WriteStream extends Writable {
+  constructor (core) {
+    super()
+    this.core = core
+  }
+
+  _writev (batch, cb) {
+    this._writevP(batch).then(cb, cb)
+  }
+
+  async _writevP (batch) {
+    await this.core.append(batch)
+  }
+}
+
+exports.WriteStream = WriteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.2",
+  "version": "10.0.0-alpha.23",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -27,22 +27,29 @@
     "url": "https://github.com/hypercore-protocol/hypercore/issues"
   },
   "homepage": "https://github.com/hypercore-protocol/hypercore#readme",
+  "files": [
+    "index.js",
+    "lib/**.js"
+  ],
   "dependencies": {
     "@hyperswarm/secret-stream": "^5.0.0",
+    "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
-    "codecs": "^2.2.0",
-    "compact-encoding": "^2.0.0",
+    "codecs": "^3.0.0",
+    "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",
-    "flat-tree": "^1.7.0",
-    "hypercore-crypto": "^2.1.1",
+    "events": "^3.3.0",
+    "flat-tree": "^1.9.0",
+    "hypercore-crypto": "^3.1.0",
     "is-options": "^1.0.1",
     "random-access-file": "^2.1.4",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
-    "uint64le": "^1.0.0"
+    "sodium-universal": "^3.0.4",
+    "xache": "^1.0.0"
   },
   "devDependencies": {
-    "brittle": "^1.4.1",
+    "brittle": "^2.0.0",
     "hyperswarm": "next",
     "random-access-memory": "^3.1.2",
     "standard": "^16.0.3",

package/.github/workflows/test-node.yml

@@ -1,24 +0,0 @@
-name: Build Status
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-jobs:
-  build:
-    strategy:
-      matrix:
-        node-version: [14.x]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm test

package/UPGRADE.md

@@ -1,9 +0,0 @@
-# Upgrade Notes
-
-Notes for downstream developers who are upgrading their modules to new, breaking versions of hypercore.
-
-## 9.0.0
-
-- The format of signatures [has been changed](https://github.com/mafintosh/hypercore/issues/260). This is backwards-compatible (v9 can read v8 signatures), but forward-incompatible (v8 cannot read v9 signatures). If a v8 peer replicates with a v9 peer, it will emit a "REMOTE SIGNTURE INVALID" error on the replication stream.
-- The encryption ([NOISE](https://github.com/emilbayes/noise-protocol)) handshake has been changed in an backwards- and forwards-incompatible way. v8 peers can not handshake with v9 peers, and vice-versa. A NOISE-related error is emitted on the replication stream.
-- There is no way (yet) to detect whether a peer is running an incompatible version of hypercore at the replication level. One workaround for downstream developers is to include their own application-level handshake before piping to the replication stream, to communicate a "app protocol version" (maybe "v8" and "v9") and abort the connection if the peer is running an incompatible version.

package/examples/announce.js

@@ -1,19 +0,0 @@
-const Hypercore = require('../')
-const Hyperswarm = require('hyperswarm')
-
-const core = new Hypercore('./source')
-
-start()
-
-async function start () {
-  await core.ready()
-  while (core.length < 1000) {
-    await core.append('block #' + core.length)
-  }
-
-  const swarm = new Hyperswarm()
-  swarm.on('connection', socket => core.replicate(socket))
-  swarm.join(core.discoveryKey, { server: true, client: false })
-
-  console.log('Core:', core.key.toString('hex'))
-}

package/examples/basic.js

@@ -1,10 +0,0 @@
-const Hypercore = require('../')
-
-start()
-
-async function start () {
-  const core = new Hypercore('/tmp/basic')
-  await core.append(['Hello', 'World'])
-  console.log(core)
-  await core.close()
-}

package/examples/http.js

@@ -1,123 +0,0 @@
-const Hypercore = require('../')
-const streamx = require('streamx')
-const replicator = require('@hyperswarm/replicator')
-
-const core = new Hypercore('/tmp/movie')
-
-if (process.argv[2] === 'bench') bench()
-else if (process.argv[2]) importData()
-else start()
-
-class ByteStream extends streamx.Readable {
-  constructor (core, byteOffset, byteLength) {
-    super()
-
-    this.core = core
-    this.byteOffset = byteOffset
-    this.byteLength = byteLength
-    this.index = 0
-    this.range = null
-  }
-
-  async _read (cb) {
-    let data = null
-
-    if (!this.byteLength) {
-      this.push(null)
-      return cb(null)
-    }
-
-    if (this.byteOffset > 0) {
-      const [block, byteOffset] = await core.seek(this.byteOffset)
-      this.byteOffset = 0
-      this.index = block + 1
-      this._select(this.index)
-      data = (await core.get(block)).slice(byteOffset)
-    } else {
-      this._select(this.index + 1)
-      data = await core.get(this.index++)
-    }
-
-    if (data.length >= this.byteLength) {
-      data = data.slice(0, this.byteLength)
-      this.push(data)
-      this.push(null)
-    } else {
-      this.push(data)
-    }
-
-    this.byteLength -= data.length
-
-    cb(null)
-  }
-
-  _select (index) {
-    if (this.range !== null) this.range.destroy(null)
-    this.range = this.core.download({ start: index, end: index + 32, linear: true })
-  }
-
-  _destroy (cb) {
-    if (this.range) this.range.destroy(null)
-    cb(null)
-  }
-}
-
-async function bench () {
-  await core.ready()
-
-  console.time()
-  for (let i = 0; i < core.length; i++) {
-    await core.get(i)
-  }
-  console.timeEnd()
-}
-
-async function start () {
-  const http = require('http')
-  const parse = require('range-parser')
-
-  await core.ready()
-
-  core.on('download', (index) => console.log('Downloaded block #' + index))
-  core.download({ start: 0, end: 1 })
-
-  // hack until we update the replicator
-  core.ready = (cb) => cb(null)
-
-  replicator(core, {
-    discoveryKey: require('crypto').createHash('sha256').update('http').digest(),
-    announce: true,
-    lookup: true
-  })
-
-  http.createServer(function (req, res) {
-    res.setHeader('Content-Type', 'video/x-matroska')
-    res.setHeader('Accept-Ranges', 'bytes')
-
-    let s
-
-    if (req.headers.range) {
-      const range = parse(core.byteLength, req.headers.range)[0]
-      const byteLength = range.end - range.start + 1
-      res.statusCode = 206
-      res.setHeader('Content-Range', 'bytes ' + range.start + '-' + range.end + '/' + core.byteLength)
-      s = new ByteStream(core, range.start, byteLength)
-    } else {
-      s = new ByteStream(core, 0, core.byteLength)
-    }
-
-    res.setHeader('Content-Length', s.byteLength)
-    s.pipe(res, () => {})
-  }).listen(10101)
-}
-
-async function importData () {
-  const fs = require('fs')
-  const rs = fs.createReadStream(process.argv[2])
-
-  for await (const data of rs) {
-    await core.append(data)
-  }
-
-  console.log('done!', core)
-}

package/examples/lookup.js

@@ -1,20 +0,0 @@
-const Hypercore = require('../')
-const Hyperswarm = require('hyperswarm')
-
-const core = new Hypercore('./clone', process.argv[2])
-
-start()
-
-async function start () {
-  await core.ready()
-
-  const swarm = new Hyperswarm()
-  swarm.on('connection', socket => core.replicate(socket))
-  swarm.join(core.discoveryKey, { server: false, client: true })
-
-  console.log((await core.get(42)).toString())
-  console.log((await core.get(142)).toString())
-  console.log((await core.get(511)).toString())
-  console.log((await core.get(512)).toString())
-  console.log((await core.get(513)).toString())
-}

package/test/basic.js

@@ -1,78 +0,0 @@
-const test = require('brittle')
-const ram = require('random-access-memory')
-
-const Hypercore = require('../')
-const { create } = require('./helpers')
-
-test('basic', async function (t) {
-  const core = await create()
-  let appends = 0
-
-  t.is(core.length, 0)
-  t.is(core.byteLength, 0)
-  t.is(core.writable, true)
-  t.is(core.readable, true)
-
-  core.on('append', function () {
-    appends++
-  })
-
-  await core.append('hello')
-  await core.append('world')
-
-  t.is(core.length, 2)
-  t.is(core.byteLength, 10)
-  t.is(appends, 2)
-
-  t.end()
-})
-
-test('session', async function (t) {
-  const core = await create()
-
-  const session = core.session()
-
-  await session.append('test')
-  t.alike(await core.get(0), Buffer.from('test'))
-  t.alike(await session.get(0), Buffer.from('test'))
-  t.end()
-})
-
-test('close', async function (t) {
-  const core = await create()
-  await core.append('hello world')
-
-  await core.close()
-
-  try {
-    await core.get(0)
-    t.fail('core should be closed')
-  } catch {
-    t.pass('get threw correctly when core was closed')
-  }
-})
-
-test('close multiple', async function (t) {
-  const core = await create()
-  await core.append('hello world')
-
-  const ev = t.test('events')
-
-  ev.plan(4)
-
-  let i = 0
-
-  core.on('close', () => ev.is(i++, 0, 'on close'))
-  core.close().then(() => ev.is(i++, 1, 'first close'))
-  core.close().then(() => ev.is(i++, 2, 'second close'))
-  core.close().then(() => ev.is(i++, 3, 'third close'))
-
-  await ev
-})
-
-test('storage options', async function (t) {
-  const core = new Hypercore({ storage: ram })
-  await core.append('hello')
-  t.alike(await core.get(0), Buffer.from('hello'))
-  t.end()
-})

package/test/bitfield.js

@@ -1,71 +0,0 @@
-const test = require('brittle')
-const ram = require('random-access-memory')
-const Bitfield = require('../lib/bitfield')
-
-test('bitfield - set and get', async function (t) {
-  const b = await Bitfield.open(ram())
-
-  t.absent(b.get(42))
-  b.set(42, true)
-  t.ok(b.get(42))
-
-  // bigger offsets
-  t.absent(b.get(42000000))
-  b.set(42000000, true)
-  t.ok(b.get(42000000))
-
-  b.set(42000000, false)
-  t.absent(b.get(42000000))
-
-  await b.flush()
-})
-
-test('bitfield - random set and gets', async function (t) {
-  const b = await Bitfield.open(ram())
-  const set = new Set()
-
-  for (let i = 0; i < 200; i++) {
-    const idx = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)
-    b.set(idx, true)
-    set.add(idx)
-  }
-
-  for (let i = 0; i < 500; i++) {
-    const idx = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)
-    const expected = set.has(idx)
-    const val = b.get(idx)
-    if (val !== expected) {
-      t.fail('expected ' + expected + ' but got ' + val + ' at ' + idx)
-      return
-    }
-  }
-
-  for (const idx of set) {
-    const val = b.get(idx)
-    if (val !== true) {
-      t.fail('expected true but got ' + val + ' at ' + idx)
-      return
-    }
-  }
-
-  t.pass('all random set and gets pass')
-})
-
-test('bitfield - reload', async function (t) {
-  const s = ram()
-
-  {
-    const b = await Bitfield.open(s)
-    b.set(142, true)
-    b.set(40000, true)
-    b.set(1424242424, true)
-    await b.flush()
-  }
-
-  {
-    const b = await Bitfield.open(s)
-    t.ok(b.get(142))
-    t.ok(b.get(40000))
-    t.ok(b.get(1424242424))
-  }
-})