hypercore 9.12.0 → 10.0.0-alpha.11

package/test/sessions.js

@@ -0,0 +1,173 @@
+const test = require('brittle')
+const ram = require('random-access-memory')
+const crypto = require('hypercore-crypto')
+
+const Hypercore = require('../')
+
+test('sessions - can create writable sessions from a read-only core', async function (t) {
+  t.plan(5)
+
+  const keyPair = crypto.keyPair()
+  const core = new Hypercore(ram, keyPair.publicKey, {
+    valueEncoding: 'utf-8'
+  })
+  await core.ready()
+  t.absent(core.writable)
+
+  const session = core.session({ keyPair: { secretKey: keyPair.secretKey } })
+  t.ok(session.writable)
+
+  try {
+    await core.append('hello')
+    t.fail('should not have appended to the read-only core')
+  } catch {
+    t.pass('read-only core append threw correctly')
+  }
+
+  try {
+    await session.append('world')
+    t.pass('session append did not throw')
+  } catch {
+    t.fail('session append should not have thrown')
+  }
+
+  t.is(core.length, 1)
+  t.end()
+})
+
+test('sessions - writable session with custom sign function', async function (t) {
+  t.plan(5)
+
+  const keyPair = crypto.keyPair()
+  const core = new Hypercore(ram, keyPair.publicKey, {
+    valueEncoding: 'utf-8'
+  })
+  await core.ready()
+  t.absent(core.writable)
+
+  const session = core.session({ sign: signable => crypto.sign(signable, keyPair.secretKey) })
+  t.ok(session.writable)
+
+  try {
+    await core.append('hello')
+    t.fail('should not have appended to the read-only core')
+  } catch {
+    t.pass('read-only core append threw correctly')
+  }
+
+  try {
+    await session.append('world')
+    t.pass('session append did not throw')
+  } catch {
+    t.fail('session append should not have thrown')
+  }
+
+  t.is(core.length, 1)
+  t.end()
+})
+
+test('sessions - writable session with invalid keypair throws', async function (t) {
+  t.plan(2)
+
+  const keyPair1 = crypto.keyPair()
+  const keyPair2 = crypto.keyPair()
+
+  try {
+    const core = new Hypercore(ram, keyPair2.publicKey) // Create a new core in read-only mode.
+    core.session({ keyPair: keyPair1 })
+    t.fail('invalid keypair did not throw')
+  } catch {
+    t.pass('invalid keypair threw')
+  }
+
+  try {
+    const core = new Hypercore(ram, keyPair1.publicKey, { keyPair: keyPair2 }) // eslint-disable-line
+    await core.ready()
+    t.fail('invalid keypair did not throw')
+  } catch {
+    t.pass('invalid keypair threw')
+  }
+})
+
+test('sessions - auto close', async function (t) {
+  const core = new Hypercore(ram, { autoClose: true })
+
+  let closed = false
+  core.on('close', function () {
+    closed = true
+  })
+
+  const a = core.session()
+  const b = core.session()
+
+  await a.close()
+  t.absent(closed, 'not closed yet')
+
+  await b.close()
+  t.ok(closed, 'all closed')
+})
+
+test('sessions - auto close different order', async function (t) {
+  const core = new Hypercore(ram, { autoClose: true })
+
+  const a = core.session()
+  const b = core.session()
+
+  let closed = false
+  a.on('close', function () {
+    closed = true
+  })
+
+  await core.close()
+  t.absent(closed, 'not closed yet')
+
+  await b.close()
+  t.ok(closed, 'all closed')
+})
+
+test('sessions - auto close with all closing', async function (t) {
+  const core = new Hypercore(ram, { autoClose: true })
+
+  const a = core.session()
+  const b = core.session()
+
+  let closed = 0
+  a.on('close', () => closed++)
+  b.on('close', () => closed++)
+  core.on('close', () => closed++)
+
+  await Promise.all([core.close(), a.close(), b.close()])
+  t.is(closed, 3, 'all closed')
+})
+
+test('sessions - auto close when using from option', async function (t) {
+  const core1 = new Hypercore(ram, {
+    autoClose: true
+  })
+  const core2 = new Hypercore({
+    preload: () => {
+      return {
+        from: core1
+      }
+    }
+  })
+  await core2.close()
+  t.ok(core1.closed)
+})
+
+test('sessions - close with from option', async function (t) {
+  const core1 = new Hypercore(ram)
+  await core1.append('hello world')
+
+  const core2 = new Hypercore({
+    preload: () => {
+      return {
+        from: core1
+      }
+    }
+  })
+  await core2.close()
+
+  t.absent(core1.closed)
+  t.alike(await core1.get(0), Buffer.from('hello world'))
+})

package/test/storage.js

@@ -0,0 +1,31 @@
+const test = require('brittle')
+const sodium = require('sodium-universal')
+const crypto = require('hypercore-crypto')
+const RAM = require('random-access-memory')
+const Hypercore = require('..')
+
+const keyPair = crypto.keyPair(Buffer.alloc(sodium.crypto_sign_SEEDBYTES, 'seed'))
+
+const encryptionKey = Buffer.alloc(sodium.crypto_stream_KEYBYTES, 'encryption key')
+
+test('storage layout', async function (t) {
+  const core = new Hypercore(RAM, { keyPair })
+
+  for (let i = 0; i < 10000; i++) {
+    await core.append(Buffer.from([i]))
+  }
+
+  t.snapshot(core.core.blocks.storage.toBuffer().toString('base64'), 'blocks')
+  t.snapshot(core.core.tree.storage.toBuffer().toString('base64'), 'tree')
+})
+
+test('encrypted storage layout', async function (t) {
+  const core = new Hypercore(RAM, { keyPair, encryptionKey })
+
+  for (let i = 0; i < 10000; i++) {
+    await core.append(Buffer.from([i]))
+  }
+
+  t.snapshot(core.core.blocks.storage.toBuffer().toString('base64'), 'blocks')
+  t.snapshot(core.core.tree.storage.toBuffer().toString('base64'), 'tree')
+})

package/test/streams.js

@@ -1,162 +1,55 @@
-var tape = require('tape')
-var collect = require('stream-collector')
-var create = require('./helpers/create')
-var bufferFrom = require('buffer-from')
+const test = require('brittle')
 
-function test (batch = 1) {
-  tape('createReadStream to createWriteStream', function (t) {
-    var feed1 = create()
-    var feed2 = create()
+const { create } = require('./helpers')
 
-    feed1.append(['hello', 'world'], function () {
-      var r = feed1.createReadStream({ batch })
-      var w = feed2.createWriteStream()
+test('basic read stream', async function (t) {
+  const core = await create()
 
-      r.pipe(w).on('finish', function () {
-        collect(feed2.createReadStream({ batch }), function (err, data) {
-          t.error(err, 'no error')
-          t.same(data, [bufferFrom('hello'), bufferFrom('world')])
-          t.end()
-        })
-      })
-    })
-  })
+  const expected = [
+    'hello',
+    'world',
+    'verden',
+    'welt'
+  ]
 
-  tape('createReadStream with start, end', function (t) {
-    var feed = create({ valueEncoding: 'utf-8' })
+  await core.append(expected)
 
-    feed.append(['hello', 'multiple', 'worlds'], function () {
-      collect(feed.createReadStream({ start: 1, end: 2, batch }), function (err, data) {
-        t.error(err, 'no error')
-        t.same(data, ['multiple'])
-        t.end()
-      })
-    })
-  })
+  for await (const data of core.createReadStream()) {
+    t.alike(data.toString(), expected.shift())
+  }
 
-  tape('createReadStream with start, no end', function (t) {
-    var feed = create({ valueEncoding: 'utf-8' })
-
-    feed.append(['hello', 'multiple', 'worlds'], function () {
-      collect(feed.createReadStream({ start: 1, batch }), function (err, data) {
-        t.error(err, 'no error')
-        t.same(data, ['multiple', 'worlds'])
-        t.end()
-      })
-    })
-  })
-
-  tape('createReadStream with no start, end', function (t) {
-    var feed = create({ valueEncoding: 'utf-8' })
-
-    feed.append(['hello', 'multiple', 'worlds'], function () {
-      collect(feed.createReadStream({ end: 2, batch }), function (err, data) {
-        t.error(err, 'no error')
-        t.same(data, ['hello', 'multiple'])
-        t.end()
-      })
-    })
-  })
-
-  tape('createReadStream with live: true', function (t) {
-    var feed = create({ valueEncoding: 'utf-8' })
-    var expected = ['a', 'b', 'c', 'd', 'e']
-
-    t.plan(expected.length)
-
-    var rs = feed.createReadStream({ live: true, batch })
-
-    rs.on('data', function (data) {
-      t.same(data, expected.shift())
-    })
-
-    rs.on('end', function () {
-      t.fail('should never end')
-    })
-
-    feed.append('a', function () {
-      feed.append('b', function () {
-        feed.append(['c', 'd', 'e'])
-      })
-    })
-  })
-
-  tape('createReadStream with live: true after append', function (t) {
-    var feed = create({ valueEncoding: 'utf-8' })
-    var expected = ['a', 'b', 'c', 'd', 'e']
-
-    t.plan(expected.length)
-
-    feed.append(['a', 'b'], function () {
-      var rs = feed.createReadStream({ live: true, batch })
-
-      rs.on('data', function (data) {
-        t.same(data, expected.shift())
-      })
-
-      rs.on('end', function () {
-        t.fail('should never end')
-      })
-
-      feed.append(['c', 'd', 'e'])
-    })
-  })
-
-  tape('createReadStream with live: true and tail: true', function (t) {
-    var feed = create({ valueEncoding: 'utf-8' })
-    var expected = ['c', 'd', 'e']
-
-    t.plan(expected.length)
-
-    feed.append(['a', 'b'], function () {
-      var rs = feed.createReadStream({ live: true, tail: true, batch })
-
-      rs.on('data', function (data) {
-        t.same(data, expected.shift())
-      })
+  t.is(expected.length, 0)
+})
 
-      rs.on('end', function () {
-        t.fail('should never end')
-      })
+test('read stream with start / end', async function (t) {
+  const core = await create()
 
-      setImmediate(function () {
-        feed.append(['c', 'd', 'e'])
-      })
-    })
-  })
-}
+  const datas = [
+    'hello',
+    'world',
+    'verden',
+    'welt'
+  ]
 
-tape('createWriteStream with maxBlockSize', function (t) {
-  t.plan(11 * 2 + 1)
+  await core.append(datas)
 
-  var feed = create()
+  {
+    const expected = datas.slice(1)
 
-  var ws = feed.createWriteStream({ maxBlockSize: 100 * 1024 })
+    for await (const data of core.createReadStream({ start: 1 })) {
+      t.alike(data.toString(), expected.shift())
+    }
 
-  ws.write(Buffer.alloc(1024 * 1024))
-  ws.end(function () {
-    t.same(feed.length, 11)
+    t.is(expected.length, 0)
+  }
 
-    sameSize(0, 100 * 1024)
-    sameSize(1, 100 * 1024)
-    sameSize(2, 100 * 1024)
-    sameSize(3, 100 * 1024)
-    sameSize(4, 100 * 1024)
-    sameSize(5, 100 * 1024)
-    sameSize(6, 100 * 1024)
-    sameSize(7, 100 * 1024)
-    sameSize(8, 100 * 1024)
-    sameSize(9, 100 * 1024)
-    sameSize(10, 1024 * 1024 - 10 * 100 * 1024)
+  {
+    const expected = datas.slice(2, 3)
 
-    function sameSize (idx, size) {
-      feed.get(idx, function (err, blk) {
-        t.error(err, 'no error')
-        t.same(blk.length, size)
-      })
+    for await (const data of core.createReadStream({ start: 2, end: 3 })) {
+      t.alike(data.toString(), expected.shift())
     }
-  })
-})
 
-test()
-test(10)
+    t.is(expected.length, 0)
+  }
+})

package/test/user-data.js

@@ -0,0 +1,47 @@
+const test = require('brittle')
+const Hypercore = require('../')
+const tmp = require('tmp-promise')
+const { create } = require('./helpers')
+
+test('userdata - can set through setUserData', async function (t) {
+  const core = await create()
+  await core.setUserData('hello', Buffer.from('world'))
+
+  t.alike(await core.getUserData('hello'), Buffer.from('world'))
+
+  t.end()
+})
+
+test('userdata - can set through constructor option', async function (t) {
+  const core = await create({
+    userData: {
+      hello: Buffer.from('world')
+    }
+  })
+
+  t.alike(await core.getUserData('hello'), Buffer.from('world'))
+
+  t.end()
+})
+
+test('userdata - persists across restarts', async function (t) {
+  const dir = await tmp.dir()
+
+  let core = new Hypercore(dir.path, {
+    userData: {
+      hello: Buffer.from('world')
+    }
+  })
+  await core.ready()
+
+  await core.close()
+  core = new Hypercore(dir.path, {
+    userData: {
+      other: Buffer.from('another')
+    }
+  })
+
+  t.alike(await core.getUserData('hello'), Buffer.from('world'))
+  t.alike(await core.getUserData('other'), Buffer.from('another'))
+  t.end()
+})

package/bench/all.sh

@@ -1,65 +0,0 @@
-run () {
-  echo '> node write-16kb-blocks.js'
-  node write-16kb-blocks.js
-  echo
-  echo '> node write-512b-blocks.js'
-  node write-512b-blocks.js
-  echo
-  echo '> node write-64kb-blocks-static.js'
-  node write-64kb-blocks-static.js
-  echo
-  echo '> node write-64kb-blocks.js'
-  node write-64kb-blocks.js
-  echo
-  echo '> node copy-64kb-blocks.js'
-  node copy-64kb-blocks.js
-  echo
-  echo '> node read-16kb-blocks-proof.js'
-  node read-16kb-blocks-proof.js
-  echo
-  echo '> node read-16kb-blocks.js'
-  node read-16kb-blocks.js
-  echo
-  echo '> node read-512b-blocks.js'
-  node read-512b-blocks.js
-  echo
-  echo '> node read-64kb-blocks-linear.js'
-  node read-64kb-blocks-linear.js
-  echo
-  echo '> node read-64kb-blocks-linear-batch.js'
-  node read-64kb-blocks-linear-batch.js
-  echo
-  echo '> node read-64kb-blocks-proof.js'
-  node read-64kb-blocks-proof.js
-  echo
-  echo '> node read-64kb-blocks.js'
-  node read-64kb-blocks.js
-  echo
-  echo '> node replicate-16kb-blocks.js'
-  node replicate-16kb-blocks.js
-  echo
-  echo '> node replicate-64kb-blocks.js'
-  node replicate-64kb-blocks.js
-  echo
-}
-
-clear_cache () {
-  echo '> rm -rf cores'
-  rm -rf cores
-  echo
-}
-
-echo "> git checkout $(git log --pretty=format:'%h' -n 1)"
-echo
-
-echo '# clearing cache'
-echo
-clear_cache
-
-echo '# running all benchmarks'
-echo
-run
-
-echo '# re-running all benchmarks'
-echo
-run

package/bench/copy-64kb-blocks.js

@@ -1,51 +0,0 @@
-var hypercore = require('../')
-var shuffle = require('shuffle-array')
-var path = require('path')
-
-var source = hypercore(path.join(__dirname, 'cores/64kb'))
-
-source.ready(function () {
-  var dest = hypercore(path.join(__dirname, 'cores/64kb-copy'), source.key, { overwrite: true })
-
-  var then = Date.now()
-  var missing = []
-  var active = 0
-  var size = 0
-  var cnt = 0
-
-  for (var i = 0; i < source.length; i++) missing.push(i)
-
-  shuffle(missing)
-
-  for (var j = 0; j < 16; j++) {
-    active++
-    copy(null, null)
-  }
-
-  function copy (err, data) {
-    if (err) throw err
-
-    active--
-
-    if (!missing.length) {
-      if (!active) {
-        console.log(Math.floor(1000 * size / (Date.now() - then)) + ' bytes/s')
-        console.log(Math.floor(1000 * cnt / (Date.now() - then)) + ' blocks/s')
-      }
-      return
-    }
-
-    var block = missing.shift()
-
-    active++
-    source.proof(block, function (err, proof) {
-      if (err) throw err
-      source.get(block, function (err, data) {
-        if (err) throw err
-        size += data.length
-        cnt++
-        dest.put(block, data, proof, copy)
-      })
-    })
-  }
-})

package/bench/helpers/read-throttled.js

@@ -1,27 +0,0 @@
-var path = require('path')
-var speedometer = require('speedometer')
-var hypercore = require('../../')
-
-module.exports = function (dir, proof) {
-  var speed = speedometer()
-  var feed = hypercore(path.join(__dirname, '../cores', dir))
-
-  feed.ready(function () {
-    for (var i = 0; i < 16; i++) read()
-
-    function read (err, data) {
-      if (speed() > 10000000) return setTimeout(read, 250)
-      if (err) throw err
-      if (data) speed(data.length)
-
-      var next = Math.floor(Math.random() * feed.length)
-      feed.get(next, read)
-      if (proof) feed.proof(next, noop)
-    }
-  })
-
-  process.title = 'hypercore-read-10mb'
-  console.log('Reading data at ~10mb/s. Pid is %d', process.pid)
-}
-
-function noop () {}

package/bench/helpers/read.js

@@ -1,47 +0,0 @@
-var path = require('path')
-var hypercore = require('../../')
-
-module.exports = function (dir, proof) {
-  var feed = hypercore(path.join(__dirname, '../cores', dir))
-
-  var then = Date.now()
-  var size = 0
-  var cnt = 0
-
-  feed.ready(function () {
-    var missing = feed.length
-    var reading = 0
-
-    for (var i = 0; i < 16; i++) read(null, null)
-
-    function read (err, data) {
-      if (err) throw err
-
-      if (data) {
-        reading--
-        cnt++
-        size += data.length
-      }
-
-      if (!missing) {
-        if (!reading) {
-          console.log(Math.floor(1000 * size / (Date.now() - then)) + ' bytes/s')
-          console.log(Math.floor(1000 * cnt / (Date.now() - then)) + ' blocks/s')
-        }
-        return
-      }
-
-      missing--
-      reading++
-
-      var block = Math.floor(Math.random() * feed.length)
-
-      if (proof) feed.proof(block, onproof)
-      else onproof()
-
-      function onproof () {
-        feed.get(block, read)
-      }
-    }
-  })
-}

package/bench/helpers/write.js

@@ -1,29 +0,0 @@
-// NOTES: rerunning this benchmark is *a lot* faster on the 2nd
-// run. Can we gain massive perf by preallocating files?
-
-var hypercore = require('../../')
-var path = require('path')
-var bufferAlloc = require('buffer-alloc-unsafe')
-
-module.exports = function (dir, blockSize, count, finalize) {
-  var feed = hypercore(path.join(__dirname, '../cores', dir), { live: !finalize, overwrite: true })
-
-  var then = Date.now()
-  var buf = bufferAlloc(blockSize)
-  buf.fill(0)
-
-  var blocks = []
-  while (blocks.length < 128) blocks.push(buf)
-
-  feed.append(blocks, function loop (err) {
-    if (err) throw err
-    if (feed.length < count) feed.append(blocks, loop)
-    else if (finalize) feed.finalize(done)
-    else done()
-  })
-
-  function done () {
-    console.log(Math.floor(1000 * blockSize * feed.length / (Date.now() - then)) + ' bytes/s')
-    console.log(Math.floor(1000 * feed.length / (Date.now() - then)) + ' blocks/s')
-  }
-}

package/bench/read-16kb-blocks-proof-throttled.js

@@ -1 +0,0 @@
-require('./helpers/read-throttled')('16kb', true)

package/bench/read-16kb-blocks-proof.js

@@ -1 +0,0 @@
-require('./helpers/read')('16kb', true)

package/bench/read-16kb-blocks-throttled.js

@@ -1 +0,0 @@
-require('./helpers/read-throttled')('16kb')

package/bench/read-16kb-blocks.js

@@ -1 +0,0 @@
-require('./helpers/read')('16kb')

package/bench/read-512b-blocks.js

@@ -1 +0,0 @@
-require('./helpers/read')('512b')