hypercore 9.12.0 → 10.0.0-alpha.11

package/test/replicate.js

@@ -1,969 +1,372 @@
-var create = require('./helpers/create')
-var replicate = require('./helpers/replicate')
-var tape = require('tape')
-var Protocol = require('hypercore-protocol')
+const test = require('brittle')
+const NoiseSecretStream = require('@hyperswarm/secret-stream')
+const { create, replicate, eventFlush } = require('./helpers')
 
-tape('replicate', function (t) {
-  t.plan(10)
+test('basic replication', async function (t) {
+  const a = await create()
 
-  var feed = create()
+  await a.append(['a', 'b', 'c', 'd', 'e'])
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  const b = await create(a.key)
 
-    clone.get(0, same(t, 'a'))
-    clone.get(1, same(t, 'b'))
-    clone.get(2, same(t, 'c'))
-    clone.get(3, same(t, 'd'))
-    clone.get(4, same(t, 'e'))
+  let d = 0
+  b.on('download', () => d++)
 
-    replicate(feed, clone, { live: true })
-  })
-})
-
-tape('replicate twice', function (t) {
-  t.plan(20)
-
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
-
-    clone.get(0, same(t, 'a'))
-    clone.get(1, same(t, 'b'))
-    clone.get(2, same(t, 'c'))
-    clone.get(3, same(t, 'd'))
-    clone.get(4, same(t, 'e'))
-
-    replicate(feed, clone).on('end', function () {
-      feed.append(['f', 'g', 'h', 'i', 'j'], function () {
-        replicate(feed, clone).on('end', function () {
-          clone.get(5, same(t, 'f'))
-          clone.get(6, same(t, 'g'))
-          clone.get(7, same(t, 'h'))
-          clone.get(8, same(t, 'i'))
-          clone.get(9, same(t, 'j'))
-        })
-      })
-    })
-  })
-})
+  replicate(a, b, t)
 
-tape('replicate live', function (t) {
-  t.plan(6)
+  const r = b.download({ start: 0, end: a.length })
 
-  var feed = create()
+  await r.downloaded()
 
-  feed.ready(function () {
-    var clone = create(feed.key)
-
-    replicate(feed, clone, { live: true })
-
-    feed.append('a')
-    feed.append('b')
-    feed.append('c')
-
-    clone.get(0, same(t, 'a'))
-    clone.get(1, same(t, 'b'))
-    clone.get(2, same(t, 'c'))
-  })
+  t.is(d, 5)
 })
 
-tape('download while get', function (t) {
-  t.plan(10)
-
-  var feed = create()
+test('basic replication from fork', async function (t) {
+  const a = await create()
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+  await a.truncate(4)
+  await a.append('e')
 
-    // add 5 so this never finished
-    clone.download({ start: 0, end: 6 }, function () {
-      t.fail('should never happen')
-    })
+  t.is(a.fork, 1)
 
-    clone.get(0, same(t, 'a'))
-    clone.get(1, same(t, 'b'))
-    clone.get(2, same(t, 'c'))
-    clone.get(3, same(t, 'd'))
-    clone.get(4, same(t, 'e'))
+  const b = await create(a.key)
 
-    replicate(feed, clone, { live: true })
-  })
-})
+  replicate(a, b, t)
 
-tape('non live', function (t) {
-  t.plan(10)
+  let d = 0
+  b.on('download', () => d++)
 
-  var feed = create()
+  const r = b.download({ start: 0, end: a.length })
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  await r.downloaded()
 
-    replicate(clone, feed).on('end', function () {
-      clone.get(0, same(t, 'a'))
-      clone.get(1, same(t, 'b'))
-      clone.get(2, same(t, 'c'))
-      clone.get(3, same(t, 'd'))
-      clone.get(4, same(t, 'e'))
-    })
-  })
+  t.is(d, 5)
+  t.is(a.fork, b.fork)
 })
 
-tape('non live, two way', function (t) {
-  t.plan(20)
-
-  var feed = create()
+test('eager replication from bigger fork', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
-
-    replicate(clone, feed).on('end', function () {
-      clone.get(0, same(t, 'a'))
-      clone.get(1, same(t, 'b'))
-      clone.get(2, same(t, 'c'))
-      clone.get(3, same(t, 'd'))
-      clone.get(4, same(t, 'e'))
-
-      var clone2 = create(feed.key)
-
-      replicate(clone, clone2).on('end', function () {
-        clone2.get(0, same(t, 'a'))
-        clone2.get(1, same(t, 'b'))
-        clone2.get(2, same(t, 'c'))
-        clone2.get(3, same(t, 'd'))
-        clone2.get(4, same(t, 'e'))
-      })
-    })
-  })
-})
+  replicate(a, b, t)
 
-tape('non-live empty', function (t) {
-  var feed = create()
+  await a.append(['a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'j', 'k'])
+  await a.truncate(4)
+  await a.append(['FORKED', 'g', 'h', 'i', 'j', 'k'])
 
-  feed.ready(function () {
-    var clone = create(feed.key)
+  t.is(a.fork, 1)
 
-    replicate(feed, clone).on('end', function () {
-      t.same(clone.length, 0)
-      t.end()
-    })
+  let d = 0
+  b.on('download', (index) => {
+    d++
   })
-})
 
-tape('basic 3-way replication', function (t) {
-  var feed = create()
+  const r = b.download({ start: 0, end: a.length })
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone1 = create(feed.key)
-    var clone2 = create(feed.key)
+  await r.downloaded()
 
-    replicate(feed, clone1, { live: true })
-    replicate(clone1, clone2, { live: true })
-
-    clone1.get(0, function (err, data) {
-      t.error(err, 'no error')
-      t.same(data, Buffer.from('a'))
-
-      clone2.get(0, function (err) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('a'))
-        t.end()
-      })
-    })
-  })
+  t.is(d, a.length)
+  t.is(a.fork, b.fork)
 })
 
-tape('basic 3-way replication sparse and not sparse', function (t) {
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone1 = create(feed.key, { sparse: true })
-    var clone2 = create(feed.key)
-
-    replicate(feed, clone1, { live: true })
+test('eager replication of updates per default', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
 
-    clone1.get(0, function (err, data) {
-      t.error(err, 'no error')
-      t.same(data, Buffer.from('a'))
+  replicate(a, b, t)
 
-      replicate(clone1, clone2, { live: true })
-
-      clone2.get(0, function (err) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('a'))
-        var inflight = clone2.peers[0].inflightRequests
-        if (inflight.length === 1 && inflight[0].index === 0) inflight = [] // just has not been cleared yet
-        t.same(inflight, [], 'no additional requests')
-        t.end()
-      })
+  const appended = new Promise(resolve => {
+    b.on('append', function () {
+      t.pass('appended')
+      resolve()
     })
   })
-})
 
-tape('extra data + factor of two', function (t) {
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], function () {
-    var clone1 = create(feed.key)
-
-    replicate(feed, clone1, { live: true })
-
-    clone1.get(1, function (err, data) {
-      t.error(err, 'no error')
-      t.same(data, Buffer.from('b'))
-      t.end()
-    })
-  })
+  await a.append(['a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'j', 'k'])
+  await appended
 })
 
-tape('3-way another index', function (t) {
-  var feed = create()
+test('bigger download range', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
 
-  feed.append(['a', 'b'], function () {
-    var clone1 = create(feed.key)
-    var clone2 = create(feed.key)
+  replicate(a, b, t)
 
-    replicate(feed, clone1, { live: true })
-    replicate(clone1, clone2, { live: true })
+  for (let i = 0; i < 20; i++) await a.append('data')
 
-    clone1.get(1, function (err, data) {
-      t.error(err, 'no error')
-      t.same(data, Buffer.from('b'))
+  const downloaded = new Set()
 
-      clone2.get(1, function (err) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('b'))
-        t.end()
-      })
-    })
+  b.on('download', function (index) {
+    downloaded.add(index)
   })
-})
-
-tape('3-way another index + extra data', function (t) {
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone1 = create(feed.key)
-    var clone2 = create(feed.key)
 
-    replicate(feed, clone1, { live: true })
-    replicate(clone1, clone2, { live: true })
+  const r = b.download({ start: 0, end: a.length })
+  await r.downloaded()
 
-    clone1.get(1, function (err, data) {
-      t.error(err, 'no error')
-      t.same(data, Buffer.from('b'))
+  t.is(b.length, a.length, 'same length')
+  t.is(downloaded.size, a.length, 'downloaded all')
 
-      clone2.get(1, function (err) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('b'))
-        t.end()
-      })
-    })
-  })
+  t.end()
 })
 
-tape('3-way another index + extra data + factor of two', function (t) {
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], function () {
-    var clone1 = create(feed.key)
-    var clone2 = create(feed.key)
-
-    replicate(feed, clone1, { live: true })
-    replicate(clone1, clone2, { live: true })
+test('high latency reorg', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
 
-    clone1.get(1, function (err, data) {
-      t.error(err, 'no error')
-      t.same(data, Buffer.from('b'))
+  const s = replicate(a, b, t)
 
-      clone2.get(1, function (err) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('b'))
-        t.end()
-      })
-    })
-  })
-})
+  for (let i = 0; i < 50; i++) await a.append('data')
 
-tape('3-way another index + extra data + factor of two + static', function (t) {
-  var feed = create({ live: false })
-
-  feed.append(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], function () {
-    feed.finalize(function () {
-      var clone1 = create(feed.key)
-      var clone2 = create(feed.key)
-
-      replicate(feed, clone1, { live: true })
-      replicate(clone1, clone2, { live: true })
+  {
+    const r = b.download({ start: 0, end: a.length })
+    await r.downloaded()
+  }
 
-      clone1.get(1, function (err, data) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('b'))
+  s[0].destroy()
+  s[1].destroy()
 
-        clone2.get(1, function (err) {
-          t.error(err, 'no error')
-          t.same(data, Buffer.from('b'))
-          t.end()
-        })
-      })
-    })
-  })
-})
+  await a.truncate(30)
 
-tape('seek while replicating', function (t) {
-  t.plan(6)
+  for (let i = 0; i < 50; i++) await a.append('fork')
 
-  var feed = create()
+  replicate(a, b, t)
 
-  feed.ready(function () {
-    var clone = create(feed.key)
+  {
+    const r = b.download({ start: 0, end: a.length })
+    await r.downloaded()
+  }
 
-    clone.seek(9, function (err, index, offset) {
-      t.error(err, 'no error')
-      t.same(index, 2)
-      t.same(offset, 1)
-    })
+  let same = 0
 
-    clone.seek(16, function (err, index, offset) {
-      t.error(err, 'no error')
-      t.same(index, 4)
-      t.same(offset, 2)
-    })
+  for (let i = 0; i < a.length; i++) {
+    const ba = await a.get(i)
+    const bb = await b.get(i)
+    if (ba.equals(bb)) same++
+  }
 
-    feed.append(['hello'], function () {
-      feed.append(['how', 'are', 'you', 'doing', '?'], function () {
-        replicate(feed, clone, { live: true })
-      })
-    })
-  })
+  t.is(a.fork, 1)
+  t.is(a.fork, b.fork)
+  t.is(same, 80)
 })
 
-tape('non spare live replication', function (t) {
-  var feed = create()
+test('invalid signature fails', async function (t) {
+  t.plan(2)
 
-  feed.on('ready', function () {
-    feed.append(['a', 'b', 'c'], function () {
-      var clone = create(feed.key)
+  const a = await create()
+  const b = await create() // not the same key
 
-      clone.get(0, function () {
-        clone.get(1, function () {
-          clone.get(2, function () {
-            clone.once('download', function () {
-              t.pass('downloaded new block')
-              t.end()
-            })
+  b.discoveryKey = a.discoveryKey // haxx to make them swarm
 
-            feed.append('a')
-          })
-        })
-      })
+  await a.append(['a', 'b', 'c', 'd', 'e'])
 
-      replicate(feed, clone, { live: true })
-    })
-  })
-})
+  const [s1, s2] = replicate(a, b, t)
 
-tape('can wait for updates', function (t) {
-  var feed = create()
-
-  feed.on('ready', function () {
-    var clone = create(feed.key)
-
-    clone.update(function (err) {
-      t.error(err, 'no error')
-      t.same(clone.length, 3)
-      t.end()
-    })
-
-    replicate(feed, clone, { live: true }).once('duplex-channel', function () {
-      feed.append(['a', 'b', 'c'])
-    })
+  s1.on('error', (err) => {
+    t.ok(err, 'stream closed')
   })
-})
 
-tape('replicate while clearing', function (t) {
-  var feed = create()
-
-  feed.on('ready', function () {
-    var clone = create(feed.key, { sparse: true })
-
-    clone.get(1, function (err) {
-      t.error(err, 'no error')
-      feed.clear(2, function (err) {
-        t.error(err, 'no error')
-        clone.get(2, { timeout: 50 }, function (err) {
-          t.ok(err, 'had timeout error')
-          t.end()
-        })
-      })
-    })
-
-    replicate(feed, clone, { live: true }).once('duplex-channel', function () {
-      feed.append(['a', 'b', 'c'])
-    })
+  s2.on('error', (err) => {
+    t.is(err.message, 'Remote signature does not match')
   })
-})
 
-tape('replicate while cancelling', function (t) {
-  t.plan(2)
+  return new Promise((resolve) => {
+    let missing = 2
 
-  var feed = create()
+    s1.on('close', onclose)
+    s2.on('close', onclose)
 
-  feed.on('ready', function () {
-    var clone = create(feed.key, { sparse: true })
-
-    clone.on('download', function () {
-      t.fail('should not download')
-    })
-
-    feed.on('upload', function () {
-      t.pass('should upload')
-      clone.cancel(0)
-    })
-
-    clone.get(0, function (err) {
-      t.ok(err, 'expected error')
-    })
-
-    feed.append(['a', 'b', 'c'])
-
-    replicate(feed, clone, { live: true })
+    function onclose () {
+      if (--missing === 0) resolve()
+    }
   })
 })
 
-tape('allow push', function (t) {
-  t.plan(3)
-
-  var feed = create()
+test('update with zero length', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
 
-  feed.on('ready', function () {
-    var clone = create(feed.key, { sparse: true, allowPush: true })
+  replicate(a, b, t)
 
-    clone.on('download', function () {
-      t.pass('push allowed')
-    })
-
-    feed.on('upload', function () {
-      t.pass('should upload')
-      clone.cancel(0)
-    })
-
-    clone.get(0, function (err) {
-      t.ok(err, 'expected error')
-    })
-
-    feed.append(['a', 'b', 'c'])
-
-    replicate(feed, clone, { live: true })
-  })
+  await b.update() // should not hang
+  t.is(b.length, 0)
 })
 
-tape('shared stream, non live', function (t) {
-  var a = create()
-  var b = create()
+test('basic multiplexing', async function (t) {
+  const a1 = await create()
+  const a2 = await create()
 
-  a.append(['a', 'b'], function () {
-    b.append(['c', 'd'], function () {
-      var a1 = create(a.key)
-      var b1 = create(b.key)
+  const b1 = await create(a1.key)
+  const b2 = await create(a2.key)
 
-      a1.ready(function () {
-        var s = a.replicate(true)
-        b1.replicate(s)
+  const a = a1.replicate(a2.replicate(true))
+  const b = b1.replicate(b2.replicate(false))
 
-        var s1 = a1.replicate(false)
-        b.replicate(s1)
+  a.pipe(b).pipe(a)
 
-        s.pipe(s1).pipe(s)
+  await a1.append('hi')
+  t.alike(await b1.get(0), Buffer.from('hi'))
 
-        s.on('end', function () {
-          t.ok(a1.has(0))
-          t.ok(a1.has(1))
-          t.ok(b1.has(0))
-          t.ok(b1.has(1))
-          t.end()
-        })
-      })
-    })
-  })
+  await a2.append('ho')
+  t.alike(await b2.get(0), Buffer.from('ho'))
 })
 
-tape('get total downloaded chunks', function (t) {
-  var feed = create()
-  feed.append(['a', 'b', 'c', 'e'])
-  feed.on('ready', function () {
-    var clone = create(feed.key, { sparse: true })
-    clone.get(1, function (err) {
-      t.error(err, 'no error')
-      t.same(clone.downloaded(), 1)
-      t.same(clone.downloaded(0), 1)
-      t.same(clone.downloaded(2), 0)
-      t.same(clone.downloaded(0, 1), 0)
-      t.same(clone.downloaded(2, 4), 0)
-      clone.get(3, function (err) {
-        t.error(err, 'no error')
-        t.same(clone.downloaded(), 2)
-        t.same(clone.downloaded(0), 2)
-        t.same(clone.downloaded(2), 1)
-        t.same(clone.downloaded(0, 3), 1)
-        t.same(clone.downloaded(2, 4), 1)
-        t.end()
-      })
-    })
-    replicate(feed, clone, { live: true })
-  })
-})
+test('async multiplexing', async function (t) {
+  const a1 = await create()
+  const b1 = await create(a1.key)
 
-tape('feed has a range of chuncks', function (t) {
-  var feed = create()
-  feed.append(['a', 'b', 'c', 'e'])
-  feed.on('ready', function () {
-    var clone = create(feed.key, { sparse: true })
-    clone.get(0, function (err) {
-      t.error(err, 'no error')
-      clone.get(1, function (err) {
-        t.error(err, 'no error')
-        clone.get(2, function (err) {
-          t.error(err, 'no error')
-          t.ok(clone.has(1))
-          t.notOk(clone.has(3))
-          t.ok(clone.has(0, clone.length - 1))
-          t.notOk(clone.has(0, clone.length))
-          t.ok(clone.has(1, 3))
-          t.notOk(clone.has(3, 4))
-          t.end()
-        })
-      })
-    })
-    replicate(feed, clone, { live: true })
-  })
-})
+  const a = a1.replicate(true)
+  const b = b1.replicate(false)
 
-tape('feed has a large range', function (t) {
-  var feed = create()
-  feed.append(['a', 'b', 'c', 'e', 'd', 'e', 'f', 'g'])
-  feed.append(['a', 'b', 'c', 'e', 'd', 'e', 'f', 'g'])
-  feed.append(['a', 'b', 'c', 'e', 'd', 'e', 'f', 'g'])
-  feed.on('ready', function () {
-    var clone = create(feed.key, { sparse: true })
-    var count = 20
-    var gotten = 20
-    function got () {
-      gotten--
-      if (gotten === 0) {
-        t.same(clone.downloaded(), 20)
-        t.notOk(clone.has(5, 24))
-        t.notOk(clone.has(12, 24))
-        t.notOk(clone.has(20, 24))
-        t.ok(clone.has(0, 20))
-        t.ok(clone.has(3, 20))
-        t.ok(clone.has(8, 20))
-        t.ok(clone.has(19, 20))
-        t.ok(clone.has(0, 16))
-        t.ok(clone.has(3, 16))
-        t.ok(clone.has(8, 16))
-        t.end()
-      }
-    }
-    for (var i = 0; i < count; i++) {
-      clone.get(i, got)
-    }
-    replicate(feed, clone, { live: true })
-  })
-})
+  a.pipe(b).pipe(a)
 
-tape('replicate no download', function (t) {
-  var feed = create()
+  const a2 = await create()
+  await a2.append('ho')
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  const b2 = await create(a2.key)
 
-    clone.get(0, function () {
-      t.fail('Data was received')
-    })
+  // b2 doesn't replicate immediately.
+  a2.replicate(a)
+  await eventFlush()
+  b2.replicate(b)
 
-    replicate(feed, clone, { live: true }, { live: true, download: false })
+  await new Promise(resolve => b2.once('peer-add', resolve))
 
-    setTimeout(function () {
-      t.pass('No data was received')
-      t.end()
-    }, 300)
-  })
+  t.is(b2.peers.length, 1)
+  t.alike(await b2.get(0), Buffer.from('ho'))
 })
 
-tape('replicate no upload', function (t) {
-  var feed = create()
+test('multiplexing with external noise stream', async function (t) {
+  const a1 = await create()
+  const a2 = await create()
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  const b1 = await create(a1.key)
+  const b2 = await create(a2.key)
 
-    clone.get(0, function () {
-      t.fail('Data was received')
-    })
+  const n1 = new NoiseSecretStream(true)
+  const n2 = new NoiseSecretStream(false)
+  n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream)
 
-    replicate(feed, clone, { live: true, upload: false }, { live: true })
+  a1.replicate(n1)
+  a2.replicate(n1)
+  b1.replicate(n2)
+  b2.replicate(n2)
 
-    setTimeout(function () {
-      t.pass('No data was received')
-      t.end()
-    }, 300)
-  })
-})
+  await a1.append('hi')
+  t.alike(await b1.get(0), Buffer.from('hi'))
 
-tape('sparse mode, two downloads', function (t) {
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key, { sparse: true })
-
-    replicate(feed, clone)
-    clone.update(function () {
-      clone.download({ start: 0, end: 4 }, function (err) {
-        t.error(err, 'no error')
-        // next tick so selection is cleared
-        process.nextTick(function () {
-          clone.download(4, function (err) {
-            t.error(err, 'no error')
-            t.end()
-          })
-        })
-      })
-    })
-  })
+  await a2.append('ho')
+  t.alike(await b2.get(0), Buffer.from('ho'))
 })
 
-tape('peer-add and peer-remove are emitted', function (t) {
-  t.plan(5)
+test('seeking while replicating', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
 
-  var feed = create()
+  replicate(a, b, t)
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  await a.append(['hello', 'this', 'is', 'test', 'data'])
 
-    feed.on('peer-add', function (peer) {
-      t.notEquals(peer.remoteId, null)
-      t.pass('peer-add1')
-    })
-    clone.on('peer-add', function (peer) {
-      t.pass('peer-add2')
-    })
-    feed.on('peer-remove', function (peer) {
-      t.pass('peer-remove1')
-    })
-    clone.on('peer-remove', function (peer) {
-      t.pass('peer-remove2')
-    })
-
-    replicate(clone, feed)
-  })
+  t.alike(await b.seek(6), [1, 1])
 })
 
-tape('replicate with onwrite', function (t) {
-  var feed = create()
+test('multiplexing multiple times over the same stream', async function (t) {
+  const a1 = await create()
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var expected = ['a', 'b', 'c', 'd', 'e']
+  await a1.append('hi')
 
-    var clone = create(feed.key, {
-      onwrite: function (index, data, peer, cb) {
-        t.ok(peer, 'has peer')
-        t.same(expected[index], data.toString())
-        expected[index] = null
-        cb()
-      }
-    })
-
-    clone.on('sync', function () {
-      t.same(expected, [null, null, null, null, null])
-      t.end()
-    })
-
-    replicate(feed, clone, { live: true })
-  })
-})
-
-tape('replicate from very sparse', function (t) {
-  t.plan(3)
-
-  var feed = create()
-  var arr = new Array(1e3)
+  const b1 = await create(a1.key)
 
-  arr.fill('a')
-  feed.append(arr, function loop (err) {
-    if (feed.length < 1e6) return feed.append(arr, loop)
+  const n1 = new NoiseSecretStream(true)
+  const n2 = new NoiseSecretStream(false)
 
-    t.error(err, 'no error')
-    t.pass('appended ' + arr.length + ' blocks')
+  n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream)
 
-    var clone1 = create(feed.key, { sparse: true })
-    var clone2 = create(feed.key)
-    var missing = 30
-    var then = 0
+  a1.replicate(n1)
 
-    replicate(feed, clone1, { live: true })
-
-    clone2.on('download', function () {
-      if (--missing <= 0) {
-        t.pass('downloaded all in ' + (Date.now() - then) + 'ms')
-      }
-    })
+  b1.replicate(n2)
+  b1.replicate(n2)
 
-    clone1.download({ start: feed.length - 30, end: feed.length }, function () {
-      then = Date.now()
-      replicate(clone2, clone1, { live: true })
-    })
-  })
-})
+  t.ok(await b1.update(), 'update once')
+  t.absent(await a1.update(), 'writer up to date')
+  t.absent(await b1.update(), 'update again')
 
-tape('first get hash, then get block', function (t) {
-  var feed = create()
-  feed.append(['a', 'b', 'c'], function () {
-    var clone = create(feed.key, { sparse: true })
-    replicate(feed, clone, { live: true })
-
-    // fetches the hash for block #2
-    clone.seek(2, function (err) {
-      t.error(err, 'no error')
-      clone.get(2, function (err, data) {
-        t.error(err, 'no error')
-        t.same(data, Buffer.from('c'))
-        t.end()
-      })
-    })
-  })
+  t.is(b1.length, a1.length, 'same length')
+  t.end()
 })
 
-tape('destroy replication stream before handshake', function (t) {
-  var feed = create()
-  feed.append(['a', 'b', 'c'], function () {
-    var stream = feed.replicate(true)
-    stream.destroy()
-    var anotherStream = feed.replicate(true)
-    setImmediate(function () {
-      anotherStream.destroy()
-      feed.ifAvailable.ready(function () {
-        t.pass('ifAvailable still triggers')
-        t.same(feed.peers.length, 0)
-        t.end()
-      })
-    })
-  })
-})
+test('destroying a stream and re-replicating works', async function (t) {
+  const core = await create()
 
-tape('request timeouts', function (t) {
-  t.plan(4)
+  while (core.length < 33) await core.append(Buffer.from('#' + core.length))
 
-  var feed = create()
-  var stream = new Protocol(false, {
-    timeout: 100
-  })
+  const clone = await create(core.key)
 
-  feed.ready(function () {
-    var ch = stream.open(feed.key, {
-      onwant (want) {
-        t.pass('got want')
-        ch.have({ start: 0, length: 1 })
-      },
-      onrequest (request) {
-        t.same(request.index, 0, 'got request for #0')
-      }
-    })
+  let s1 = core.replicate(true)
+  let s2 = clone.replicate(false)
 
-    t.same(typeof stream.timeout.ms, 'number', 'can read timeout ms from protocol stream')
+  s1.pipe(s2).pipe(s1)
 
-    var timeout = setTimeout(() => t.fail('request should have timed out'), stream.timeout.ms * 2)
-    var feedStream = feed.replicate(true, { download: true, timeout: 100 })
-    stream.pipe(feedStream).pipe(stream)
+  await s2.opened
 
-    feedStream.on('error', function (err) {
-      clearTimeout(timeout)
-      t.ok(err, 'stream had timeout error')
-    })
+  const all = []
+  for (let i = 0; i < 33; i++) {
+    all.push(clone.get(i))
+  }
 
-    stream.on('error', () => {})
+  clone.once('download', function () {
+    // simulate stream failure in the middle of bulk downloading
+    s1.destroy()
   })
-})
 
-tape('double replicate', function (t) {
-  var feed = create()
+  await new Promise((resolve) => s1.once('close', resolve))
 
-  feed.append('hi', function () {
-    var clone = create(feed.key)
+  // retry
+  s1 = core.replicate(true)
+  s2 = clone.replicate(false)
 
-    var a = feed.replicate(true)
-    var b = clone.replicate(false)
-    var missing = 2
+  s1.pipe(s2).pipe(s1)
 
-    a.pipe(b).pipe(a)
-    feed.replicate(a) // replicate twice
+  const blocks = await Promise.all(all)
 
-    b.on('end', done)
-    a.on('end', done)
-
-    function done () {
-      if (!--missing) return
-      feed.ifAvailable.ready(function () {
-        clone.ifAvailable.ready(function () {
-          t.pass('no lingering state')
-          t.end()
-        })
-      })
-    }
-  })
+  t.is(blocks.length, 33, 'downloaded 33 blocks')
 })
 
-tape('events: replicating events fired (normal)', function (t) {
-  var feed = create()
-  t.plan(1)
-  feed.on('replicating', () => {
-    t.pass('replicating')
-  })
-  feed.replicate(true).on('end', function () {
-    t.end()
-  })
-})
+test('replicate discrete range', async function (t) {
+  const a = await create()
 
-tape('events: replicating events once for multiple feeds', function (t) {
-  var feed = create()
-  var stream = new Protocol(true)
-  t.plan(1)
-  feed.on('replicating', () => {
-    t.pass('replicating')
-  })
-  feed.replicate(stream)
-  feed.replicate(stream)
-  feed.on('end', function () {
-    t.end()
-  })
-})
-
-tape('events: replicating events not fired if canceled before ready', function (t) {
-  var feed = create()
-  var stream = new Protocol(true)
-  t.plan(0)
-  feed.on('replicating', () => {
-    t.pass('replicating')
-  })
-  stream.destroy()
-  feed.replicate(stream, { live: true })
-  t.end()
-})
+  await a.append(['a', 'b', 'c', 'd', 'e'])
 
-tape('regression: replicate without timeout', function (t) {
-  t.plan(10)
+  const b = await create(a.key)
 
-  var feed = create()
+  let d = 0
+  b.on('download', () => d++)
 
-  feed.append(['a', 'b', 'c', 'd', 'e'], function () {
-    var clone = create(feed.key)
+  replicate(a, b, t)
 
-    clone.get(0, same(t, 'a'))
-    clone.get(1, same(t, 'b'))
-    clone.get(2, same(t, 'c'))
-    clone.get(3, same(t, 'd'))
-    clone.get(4, same(t, 'e'))
+  const r = b.download({ blocks: [0, 2, 3] })
+  await r.downloaded()
 
-    replicate(feed, clone, { live: true, timeout: false })
-  })
+  t.is(d, 3)
+  t.alike(await b.get(0), Buffer.from('a'))
+  t.alike(await b.get(2), Buffer.from('c'))
+  t.alike(await b.get(3), Buffer.from('d'))
 })
 
-tape('replicate with NOISE disabled', function (t) {
-  var feed = create()
-  feed.append(['a', 'b', 'c'], function () {
-    var clone = create(feed.key)
-    const stream = replicate(feed, clone, { live: false, noise: false, encrypted: false })
-    clone.get(2, (err, data) => {
-      t.error(err, 'no error')
-      t.same(data.toString(), 'c')
-      t.same(stream.remoteVerified(feed.key), false, 'remote is not verified')
-      t.end()
-    })
-  })
-})
+test('replicate discrete empty range', async function (t) {
+  const a = await create()
 
-tape('replicate with feed authentication', function (t) {
-  var feed = create()
-  const keysA = Protocol.keyPair()
-  const keysB = Protocol.keyPair()
-  t.plan(5)
-  feed.ready(function (err) {
-    t.error(err)
-    var clone = create(feed.key)
-    const stream = feed.replicate(true, {
-      keyPair: keysA,
-      onfeedauthenticate: function (authFeed, publicKey, cb) {
-        t.equals(authFeed, feed)
-        t.same(publicKey, keysB.publicKey)
-        cb()
-      }
-    })
-    stream.pipe(clone.replicate(false, {
-      keyPair: keysB,
-      onfeedauthenticate: function (authFeed, publicKey, cb) {
-        t.equals(authFeed, clone)
-        t.same(publicKey, keysA.publicKey)
-        cb()
-      }
-    })).pipe(stream)
-      .on('end', function () {
-        t.end()
-      })
-  })
-})
+  await a.append(['a', 'b', 'c', 'd', 'e'])
 
-tape('replicate and close through stream', function (t) {
-  var feed = create()
-  var streams
-  var clone
-
-  feed.append(['a', 'b', 'c'], function () {
-    clone = create(feed.key)
-    streams = [feed.replicate(true, { live: true }), clone.replicate(false, { live: true })]
-    streams[0].pipe(streams[1]).pipe(streams[0])
-    streams[0].on('error', () => {})
-    streams[1].on('error', () => {})
-  })
+  const b = await create(a.key)
 
-  feed.once('peer-open', function () {
-    streams[0].close(feed.discoveryKey)
-    streams[0].destroy()
-    streams[0].on('close', function () {
-      t.same(feed.peers.length, 0)
-      t.end()
-    })
-  })
-})
+  let d = 0
+  b.on('download', () => d++)
 
-tape('download blocks', function (t) {
-  var feed = create()
-
-  feed.append(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'], function () {
-    var clone = create(feed.key, { sparse: true })
-
-    clone.download({ start: 0, end: 10, blocks: [0, 3, 4, 9] }, function (err) {
-      t.error(err, 'no error')
-      t.same(clone.length, 10)
-      t.ok(clone.has(0))
-      t.notOk(clone.has(1))
-      t.notOk(clone.has(2))
-      t.ok(clone.has(3))
-      t.ok(clone.has(4))
-      t.notOk(clone.has(5))
-      t.notOk(clone.has(6))
-      t.notOk(clone.has(7))
-      t.notOk(clone.has(8))
-      t.ok(clone.has(9))
-      t.end()
-    })
+  replicate(a, b, t)
 
-    replicate(feed, clone, { live: true })
-  })
-})
+  const r = b.download({ blocks: [] })
+  await r.downloaded()
 
-function same (t, val) {
-  return function (err, data) {
-    t.error(err, 'no error')
-    t.same(data.toString(), val)
-  }
-}
+  t.is(d, 0)
+})