hypercore 9.12.0 → 10.0.0-alpha.11

package/test/bitfield.js

@@ -1,156 +1,71 @@
-var tape = require('tape')
-var bitfield = require('../lib/bitfield')
+const test = require('brittle')
+const ram = require('random-access-memory')
+const Bitfield = require('../lib/bitfield')
 
-tape('set and get', function (t) {
-  var b = bitfield()
+test('bitfield - set and get', async function (t) {
+  const b = await Bitfield.open(ram())
 
-  t.same(b.get(0), false)
-  t.same(b.set(0, true), true)
-  t.same(b.set(0, true), false)
-  t.same(b.get(0), true)
+  t.absent(b.get(42))
+  b.set(42, true)
+  t.ok(b.get(42))
 
-  t.same(b.get(1424244), false)
-  t.same(b.set(1424244, true), true)
-  t.same(b.set(1424244, true), false)
-  t.same(b.get(1424244), true)
+  // bigger offsets
+  t.absent(b.get(42000000))
+  b.set(42000000, true)
+  t.ok(b.get(42000000))
 
-  t.end()
-})
-
-tape('set and get (tree)', function (t) {
-  var b = bitfield()
-  var tree = b.tree
-
-  t.same(tree.get(0), false)
-  t.same(tree.set(0, true), true)
-  t.same(tree.set(0, true), false)
-  t.same(tree.get(0), true)
-
-  t.same(tree.get(1424244), false)
-  t.same(tree.set(1424244, true), true)
-  t.same(tree.set(1424244, true), false)
-  t.same(tree.get(1424244), true)
-
-  t.same(b.get(0), false)
-  t.same(b.get(1424244), false)
-
-  t.end()
-})
-
-tape('set and index', function (t) {
-  var b = bitfield()
-  var ite = b.iterator(0, 100000000)
-  var i = 0
-
-  t.same(ite.next(), 0)
-
-  b.set(0, true)
-  t.same(ite.seek(0).next(), 1)
-
-  b.set(479, true)
-  t.same(ite.seek(478).next(), 478)
-  t.same(ite.next(), 480)
-
-  b.set(1, true)
-  t.same(ite.seek(0).next(), 2)
-
-  b.set(2, true)
-  t.same(ite.seek(0).next(), 3)
-
-  b.set(3, true)
-  t.same(ite.seek(0).next(), 4)
-
-  for (i = 0; i < b.length; i++) {
-    b.set(i, true)
-  }
+  b.set(42000000, false)
+  t.absent(b.get(42000000))
 
-  t.same(ite.seek(0).next(), b.length)
-
-  for (i = 0; i < b.length; i++) {
-    b.set(i, false)
-  }
-
-  t.same(ite.seek(0).next(), 0)
-  t.end()
+  await b.flush()
 })
 
-tape('set and index (random)', function (t) {
-  var b = bitfield()
+test('bitfield - random set and gets', async function (t) {
+  const b = await Bitfield.open(ram())
+  const set = new Set()
 
-  for (var i = 0; i < 100; i++) {
-    t.ok(check(), 'index validates')
-    set(Math.round(Math.random() * 2000), Math.round(Math.random() * 8))
+  for (let i = 0; i < 200; i++) {
+    const idx = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)
+    b.set(idx, true)
+    set.add(idx)
   }
 
-  t.ok(check(), 'index validates')
-  t.end()
-
-  function check () {
-    var all = []
-    var ite = b.iterator()
-    var i = 0
-
-    for (i = 0; i < b.length; i++) {
-      all[i] = true
-    }
-
-    i = ite.next()
-
-    while (i > -1) {
-      all[i] = false
-      i = ite.next()
+  for (let i = 0; i < 500; i++) {
+    const idx = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)
+    const expected = set.has(idx)
+    const val = b.get(idx)
+    if (val !== expected) {
+      t.fail('expected ' + expected + ' but got ' + val + ' at ' + idx)
+      return
     }
-
-    for (i = 0; i < b.length; i++) {
-      if (b.get(i) !== all[i]) {
-        return false
-      }
-    }
-
-    return true
   }
 
-  function set (i, n) {
-    while (n--) {
-      b.set(i++, true)
+  for (const idx of set) {
+    const val = b.get(idx)
+    if (val !== true) {
+      t.fail('expected true but got ' + val + ' at ' + idx)
+      return
     }
   }
-})
 
-tape('get total positive bits', function (t) {
-  var b = bitfield()
-
-  t.same(b.set(1, true), true)
-  t.same(b.set(2, true), true)
-  t.same(b.set(4, true), true)
-  t.same(b.set(5, true), true)
-  t.same(b.set(39, true), true)
-
-  t.same(b.total(0, 4), 2)
-  t.same(b.total(3, 4), 0)
-  t.same(b.total(3, 5), 1)
-  t.same(b.total(3, 40), 3)
-  t.same(b.total(), 5)
-  t.same(b.total(7), 1)
-
-  t.end()
+  t.pass('all random set and gets pass')
 })
 
-tape('bitfield dedup', function (t) {
-  var b = bitfield()
+test('bitfield - reload', async function (t) {
+  const s = ram()
 
-  for (var i = 0; i < 32 * 1024; i++) {
-    b.set(i, true)
+  {
+    const b = await Bitfield.open(s)
+    b.set(142, true)
+    b.set(40000, true)
+    b.set(1424242424, true)
+    await b.flush()
   }
 
-  for (var j = 0; j < 64 * 1024; j++) {
-    b.tree.set(j, true)
+  {
+    const b = await Bitfield.open(s)
+    t.ok(b.get(142))
+    t.ok(b.get(40000))
+    t.ok(b.get(1424242424))
   }
-
-  t.ok(b.get(8 * 1024))
-  t.ok(b.get(16 * 1024))
-  b.set(8 * 1024, false)
-  t.notOk(b.get(8 * 1024))
-  t.ok(b.get(16 * 1024))
-  t.end()
 })

package/test/core.js

@@ -0,0 +1,290 @@
+const test = require('brittle')
+const RAM = require('random-access-memory')
+const Core = require('../lib/core')
+
+test('core - append', async function (t) {
+  const { core } = await create()
+
+  {
+    const seq = await core.append([
+      Buffer.from('hello'),
+      Buffer.from('world')
+    ])
+
+    t.is(seq, 0)
+    t.is(core.tree.length, 2)
+    t.is(core.tree.byteLength, 10)
+    t.alike([
+      await core.blocks.get(0),
+      await core.blocks.get(1)
+    ], [
+      Buffer.from('hello'),
+      Buffer.from('world')
+    ])
+  }
+
+  {
+    const seq = await core.append([
+      Buffer.from('hej')
+    ])
+
+    t.is(seq, 2)
+    t.is(core.tree.length, 3)
+    t.is(core.tree.byteLength, 13)
+    t.alike([
+      await core.blocks.get(0),
+      await core.blocks.get(1),
+      await core.blocks.get(2)
+    ], [
+      Buffer.from('hello'),
+      Buffer.from('world'),
+      Buffer.from('hej')
+    ])
+  }
+})
+
+test('core - append and truncate', async function (t) {
+  const { core, reopen } = await create()
+
+  await core.append([
+    Buffer.from('hello'),
+    Buffer.from('world'),
+    Buffer.from('fo'),
+    Buffer.from('ooo')
+  ])
+
+  await core.truncate(3, 1)
+
+  t.is(core.tree.length, 3)
+  t.is(core.tree.byteLength, 12)
+  t.is(core.tree.fork, 1)
+  t.alike(core.header.hints.reorgs, [{ from: 0, to: 1, ancestors: 3 }])
+
+  await core.append([
+    Buffer.from('a'),
+    Buffer.from('b'),
+    Buffer.from('c'),
+    Buffer.from('d')
+  ])
+
+  await core.truncate(3, 2)
+
+  t.is(core.tree.length, 3)
+  t.is(core.tree.byteLength, 12)
+  t.is(core.tree.fork, 2)
+  t.alike(core.header.hints.reorgs, [{ from: 0, to: 1, ancestors: 3 }, { from: 1, to: 2, ancestors: 3 }])
+
+  await core.truncate(2, 3)
+
+  t.alike(core.header.hints.reorgs, [{ from: 2, to: 3, ancestors: 2 }])
+
+  await core.append([Buffer.from('a')])
+  await core.truncate(2, 4)
+
+  await core.append([Buffer.from('a')])
+  await core.truncate(2, 5)
+
+  await core.append([Buffer.from('a')])
+  await core.truncate(2, 6)
+
+  await core.append([Buffer.from('a')])
+  await core.truncate(2, 7)
+
+  t.is(core.header.hints.reorgs.length, 4)
+
+  // check that it was persisted
+  const coreReopen = await reopen()
+
+  t.is(coreReopen.tree.length, 2)
+  t.is(coreReopen.tree.byteLength, 10)
+  t.is(coreReopen.tree.fork, 7)
+  t.is(coreReopen.header.hints.reorgs.length, 4)
+})
+
+test('core - user data', async function (t) {
+  const { core, reopen } = await create()
+
+  await core.userData('hello', Buffer.from('world'))
+  t.alike(core.header.userData, [{ key: 'hello', value: Buffer.from('world') }])
+
+  await core.userData('hej', Buffer.from('verden'))
+  t.alike(core.header.userData, [
+    { key: 'hello', value: Buffer.from('world') },
+    { key: 'hej', value: Buffer.from('verden') }
+  ])
+
+  await core.userData('hello', null)
+  t.alike(core.header.userData, [{ key: 'hej', value: Buffer.from('verden') }])
+
+  await core.userData('hej', Buffer.from('world'))
+  t.alike(core.header.userData, [{ key: 'hej', value: Buffer.from('world') }])
+
+  // check that it was persisted
+  const coreReopen = await reopen()
+
+  t.alike(coreReopen.header.userData, [{ key: 'hej', value: Buffer.from('world') }])
+})
+
+test('core - verify', async function (t) {
+  const { core } = await create()
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.signer.publicKey } })
+
+  t.is(clone.header.signer.publicKey, core.header.signer.publicKey)
+
+  await core.append([Buffer.from('a'), Buffer.from('b')])
+
+  {
+    const p = await core.tree.proof({ upgrade: { start: 0, length: 2 } })
+    await clone.verify(p)
+  }
+
+  t.is(clone.header.tree.length, 2)
+  t.is(clone.header.tree.signature, core.header.tree.signature)
+
+  {
+    const p = await core.tree.proof({ block: { index: 1, nodes: await clone.tree.nodes(2), value: true } })
+    p.block.value = await core.blocks.get(1)
+    await clone.verify(p)
+  }
+})
+
+test('core - verify parallel upgrades', async function (t) {
+  const { core } = await create()
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.signer.publicKey } })
+
+  t.is(clone.header.signer.publicKey, core.header.signer.publicKey)
+
+  await core.append([Buffer.from('a'), Buffer.from('b'), Buffer.from('c'), Buffer.from('d')])
+
+  {
+    const p1 = await core.tree.proof({ upgrade: { start: 0, length: 2 } })
+    const p2 = await core.tree.proof({ upgrade: { start: 0, length: 3 } })
+
+    const v1 = clone.verify(p1)
+    const v2 = clone.verify(p2)
+
+    await v1
+    await v2
+  }
+
+  t.is(clone.header.tree.length, core.header.tree.length)
+  t.is(clone.header.tree.signature, core.header.tree.signature)
+})
+
+test('core - update hook is triggered', async function (t) {
+  const { core } = await create()
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.signer.publicKey } })
+
+  let ran = 0
+
+  core.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b01, 'was appended')
+    t.is(from, null, 'was local')
+    t.alike(bitfield, { drop: false, start: 0, length: 4 })
+    ran |= 1
+  }
+
+  await core.append([Buffer.from('a'), Buffer.from('b'), Buffer.from('c'), Buffer.from('d')])
+
+  const peer = {}
+
+  clone.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b01, 'was appended')
+    t.is(from, peer, 'was remote')
+    t.alike(bitfield, { drop: false, start: 1, length: 1 })
+    t.alike(value, Buffer.from('b'))
+    ran |= 2
+  }
+
+  {
+    const p = await core.tree.proof({ block: { index: 1, nodes: 0, value: true }, upgrade: { start: 0, length: 2 } })
+    p.block.value = await core.blocks.get(1)
+    await clone.verify(p, peer)
+  }
+
+  clone.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b00, 'no append or truncate')
+    t.is(from, peer, 'was remote')
+    t.alike(bitfield, { drop: false, start: 3, length: 1 })
+    t.alike(value, Buffer.from('d'))
+    ran |= 4
+  }
+
+  {
+    const p = await core.tree.proof({ block: { index: 3, nodes: await clone.tree.nodes(6), value: true } })
+    p.block.value = await core.blocks.get(3)
+    await clone.verify(p, peer)
+  }
+
+  core.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b10, 'was truncated')
+    t.is(from, null, 'was local')
+    t.alike(bitfield, { drop: true, start: 1, length: 3 })
+    ran |= 8
+  }
+
+  await core.truncate(1, 1)
+
+  core.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b01, 'was appended')
+    t.is(from, null, 'was local')
+    t.alike(bitfield, { drop: false, start: 1, length: 1 })
+    ran |= 16
+  }
+
+  await core.append([Buffer.from('e')])
+
+  clone.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b11, 'was appended and truncated')
+    t.is(from, peer, 'was remote')
+    t.alike(bitfield, { drop: true, start: 1, length: 3 })
+    ran |= 32
+  }
+
+  {
+    const p = await core.tree.proof({ block: { index: 0, nodes: 0, value: false }, upgrade: { start: 0, length: 2 } })
+    const r = await clone.tree.reorg(p)
+
+    await clone.reorg(r, peer)
+  }
+
+  core.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b10, 'was truncated')
+    t.is(from, null, 'was local')
+    t.alike(bitfield, { drop: true, start: 1, length: 1 })
+    ran |= 64
+  }
+
+  await core.truncate(1, 2)
+
+  clone.onupdate = (status, bitfield, value, from) => {
+    t.is(status, 0b10, 'was truncated')
+    t.is(from, peer, 'was remote')
+    t.alike(bitfield, { drop: true, start: 1, length: 1 })
+    ran |= 128
+  }
+
+  {
+    const p = await core.tree.proof({ block: { index: 0, nodes: 0, value: false }, upgrade: { start: 0, length: 1 } })
+    const r = await clone.tree.reorg(p)
+
+    await clone.reorg(r, peer)
+  }
+
+  t.is(ran, 255, 'ran all')
+})
+
+async function create (opts) {
+  const storage = new Map()
+
+  const createFile = (name) => {
+    if (storage.has(name)) return storage.get(name)
+    const s = new RAM()
+    storage.set(name, s)
+    return s
+  }
+
+  const reopen = () => Core.open(createFile, opts)
+  const core = await reopen()
+  return { core, reopen }
+}

package/test/encodings.js

@@ -0,0 +1,18 @@
+const test = require('brittle')
+const { create } = require('./helpers')
+
+test('encodings - supports built ins', async function (t) {
+  const a = await create(null, { valueEncoding: 'json' })
+
+  await a.append({ hello: 'world' })
+  t.alike(await a.get(0), { hello: 'world' })
+  t.alike(await a.get(0, { valueEncoding: 'utf-8' }), '{"hello":"world"}')
+})
+
+test('encodings - supports custom encoding', async function (t) {
+  const a = await create(null, { valueEncoding: { encode () { return Buffer.from('foo') }, decode () { return 'bar' } } })
+
+  await a.append({ hello: 'world' })
+  t.is(await a.get(0), 'bar')
+  t.alike(await a.get(0, { valueEncoding: 'utf-8' }), 'foo')
+})

package/test/encryption.js

@@ -0,0 +1,123 @@
+const test = require('brittle')
+const RAM = require('random-access-memory')
+const Hypercore = require('..')
+const { create, replicate } = require('./helpers')
+
+const encryptionKey = Buffer.alloc(32, 'hello world')
+
+test('encrypted append and get', async function (t) {
+  const a = await create({ encryptionKey })
+
+  t.alike(a.encryptionKey, encryptionKey)
+
+  await a.append(['hello'])
+
+  t.is(a.byteLength, 5)
+  t.is(a.core.tree.byteLength, 5 + a.padding)
+
+  const unencrypted = await a.get(0)
+  t.alike(unencrypted, Buffer.from('hello'))
+
+  const encrypted = await a.core.blocks.get(0)
+  t.absent(encrypted.includes('hello'))
+})
+
+test('encrypted seek', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['hello', 'world', '!'])
+
+  t.alike(await a.seek(0), [0, 0])
+  t.alike(await a.seek(4), [0, 4])
+  t.alike(await a.seek(5), [1, 0])
+  t.alike(await a.seek(6), [1, 1])
+  t.alike(await a.seek(6), [1, 1])
+  t.alike(await a.seek(9), [1, 4])
+  t.alike(await a.seek(10), [2, 0])
+  t.alike(await a.seek(11), [3, 0])
+})
+
+test('encrypted replication', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  t.test('with encryption key', async function (t) {
+    const b = await create(a.key, { encryptionKey })
+
+    replicate(a, b, t)
+
+    await t.test('through direct download', async function (t) {
+      const r = b.download({ start: 0, end: a.length })
+      await r.downloaded()
+
+      for (let i = 0; i < 5; i++) {
+        t.alike(await b.get(i), await a.get(i))
+      }
+    })
+
+    await t.test('through indirect download', async function (t) {
+      await a.append(['f', 'g', 'h', 'i', 'j'])
+
+      for (let i = 5; i < 10; i++) {
+        t.alike(await b.get(i), await a.get(i))
+      }
+    })
+  })
+
+  t.test('without encryption key', async function (t) {
+    const b = await create(a.key)
+
+    replicate(a, b, t)
+
+    await t.test('through direct download', async function (t) {
+      const r = b.download({ start: 0, end: a.length })
+      await r.downloaded()
+
+      for (let i = 0; i < 5; i++) {
+        t.alike(await b.get(i), await a.core.blocks.get(i))
+      }
+    })
+
+    await t.test('through indirect download', async function (t) {
+      await a.append(['f', 'g', 'h', 'i', 'j'])
+
+      for (let i = 5; i < 10; i++) {
+        t.alike(await b.get(i), await a.core.blocks.get(i))
+      }
+    })
+  })
+})
+
+test('encrypted session', async function (t) {
+  const a = await create({ encryptionKey })
+
+  await a.append(['hello'])
+
+  const s = a.session()
+
+  t.alike(a.encryptionKey, s.encryptionKey)
+  t.alike(await s.get(0), Buffer.from('hello'))
+
+  await s.append(['world'])
+
+  const unencrypted = await s.get(1)
+  t.alike(unencrypted, Buffer.from('world'))
+  t.alike(await a.get(1), unencrypted)
+
+  const encrypted = await s.core.blocks.get(1)
+  t.absent(encrypted.includes('world'))
+  t.alike(await a.core.blocks.get(1), encrypted)
+})
+
+test('encrypted session before ready core', async function (t) {
+  const a = new Hypercore(RAM, { encryptionKey })
+  const s = a.session()
+
+  await a.ready()
+
+  t.alike(a.encryptionKey, s.encryptionKey)
+
+  await a.append(['hello'])
+  t.alike(await s.get(0), Buffer.from('hello'))
+})

package/test/extension.js

@@ -0,0 +1,71 @@
+const test = require('brittle')
+const { create, replicate, eventFlush } = require('./helpers')
+
+test('basic extension', async function (t) {
+  const messages = ['world', 'hello']
+
+  const a = await create()
+  a.registerExtension('test-extension', {
+    encoding: 'utf-8',
+    onmessage: (message, peer) => {
+      t.ok(peer === a.peers[0])
+      t.is(message, messages.pop())
+    }
+  })
+
+  const b = await create(a.key)
+  const bExt = b.registerExtension('test-extension', {
+    encoding: 'utf-8'
+  })
+
+  replicate(a, b, t)
+
+  await eventFlush()
+  t.is(b.peers.length, 1)
+
+  bExt.send('hello', b.peers[0])
+  bExt.send('world', b.peers[0])
+
+  await eventFlush()
+  t.absent(messages.length)
+
+  t.end()
+})
+
+test('two extensions', async function (t) {
+  const messages = ['world', 'hello']
+
+  const a = await create()
+  const b = await create(a.key)
+
+  replicate(a, b, t)
+
+  b.registerExtension('test-extension-1', {
+    encoding: 'utf-8'
+  })
+  const bExt2 = b.registerExtension('test-extension-2', {
+    encoding: 'utf-8'
+  })
+
+  await eventFlush()
+  t.is(b.peers.length, 1)
+
+  bExt2.send('world', b.peers[0])
+
+  await eventFlush()
+
+  a.registerExtension('test-extension-2', {
+    encoding: 'utf-8',
+    onmessage: (message, peer) => {
+      t.ok(peer === a.peers[0])
+      t.is(message, messages.pop())
+    }
+  })
+
+  bExt2.send('hello', b.peers[0])
+
+  await eventFlush()
+  t.is(messages.length, 1) // First message gets ignored
+
+  t.end()
+})