hypercore 9.12.0 → 10.0.0-alpha.11

package/lib/streams.js

@@ -0,0 +1,39 @@
+const { Readable } = require('streamx')
+
+class ReadStream extends Readable {
+  constructor (core, opts = {}) {
+    super()
+
+    this.core = core
+    this.start = opts.start || 0
+    this.end = typeof opts.end === 'number' ? opts.end : -1
+    this.snapshot = !opts.live && opts.snapshot !== false
+    this.live = !!opts.live
+  }
+
+  _open (cb) {
+    this._openP().then(cb, cb)
+  }
+
+  _read (cb) {
+    this._readP().then(cb, cb)
+  }
+
+  async _openP () {
+    if (this.end === -1) await this.core.update()
+    else await this.core.ready()
+    if (this.snapshot && this.end === -1) this.end = this.core.length
+  }
+
+  async _readP () {
+    const end = this.live ? -1 : (this.end === -1 ? this.core.length : this.end)
+    if (end >= 0 && this.start >= end) {
+      this.push(null)
+      return
+    }
+
+    this.push(await this.core.get(this.start++))
+  }
+}
+
+exports.ReadStream = ReadStream

package/package.json

@@ -1,57 +1,56 @@
 {
   "name": "hypercore",
-  "version": "9.12.0",
-  "description": "Hypercore is a secure, distributed append-only log",
+  "version": "10.0.0-alpha.11",
+  "description": "Hypercore 10",
   "main": "index.js",
-  "dependencies": {
-    "abstract-extension": "^3.0.1",
-    "atomic-batcher": "^1.0.2",
-    "bitfield-rle": "^2.2.1",
-    "codecs": "^2.0.0",
-    "fast-bitfield": "^1.2.2",
-    "flat-tree": "^1.6.0",
-    "hypercore-cache": "^1.0.1",
-    "hypercore-crypto": "^2.0.0",
-    "hypercore-default-storage": "^1.1.1",
-    "hypercore-protocol": "^8.0.5",
-    "hypercore-streams": "^1.0.1",
-    "inherits": "^2.0.3",
-    "inspect-custom-symbol": "^1.1.0",
-    "last-one-wins": "^1.0.4",
-    "memory-pager": "^1.0.2",
-    "merkle-tree-stream": "^4.0.0",
-    "nanoguard": "^1.2.0",
-    "nanoresource": "^1.3.0",
-    "pretty-hash": "^1.0.1",
-    "sparse-bitfield": "^3.0.0",
-    "timeout-refresh": "^1.0.3",
-    "uint64be": "^2.0.1",
-    "unordered-array-remove": "^1.0.2",
-    "unordered-set": "^2.0.0"
-  },
-  "devDependencies": {
-    "buffer-from": "^1.1.1",
-    "nyc": "^14.1.1",
-    "random-access-memory": "^3.1.0",
-    "shuffle-array": "^1.0.1",
-    "speedometer": "^1.0.0",
-    "standard": "^14.3.4",
-    "stream-collector": "^1.0.1",
-    "tape": "^4.6.3"
-  },
   "scripts": {
-    "test": "standard && tape test/*.js",
-    "cov": "nyc tape test/*.js",
-    "bench": "cd bench && ./all.sh"
+    "test": "standard && brittle test/*.js"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/mafintosh/hypercore.git"
+    "url": "git+https://github.com/hypercore-protocol/hypercore.git"
   },
-  "author": "Mathias Buus (@mafintosh)",
+  "contributors": [
+    {
+      "name": "Mathias Buus",
+      "email": "mathiasbuus@gmail.com",
+      "url": "https://mafinto.sh"
+    },
+    {
+      "name": "Andrew Osheroff",
+      "email": "andrewosh@gmail.com",
+      "url": "https://andrewosh.com"
+    }
+  ],
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/mafintosh/hypercore/issues"
+    "url": "https://github.com/hypercore-protocol/hypercore/issues"
+  },
+  "homepage": "https://github.com/hypercore-protocol/hypercore#readme",
+  "dependencies": {
+    "@hyperswarm/secret-stream": "^5.0.0",
+    "b4a": "^1.1.0",
+    "big-sparse-array": "^1.0.2",
+    "codecs": "^2.2.0",
+    "compact-encoding": "^2.5.0",
+    "crc32-universal": "^1.0.1",
+    "flat-tree": "^1.9.0",
+    "hypercore-crypto": "^2.1.1",
+    "is-options": "^1.0.1",
+    "random-access-file": "^2.1.4",
+    "random-array-iterator": "^1.0.0",
+    "safety-catch": "^1.0.1",
+    "sodium-universal": "^3.0.4",
+    "xache": "^1.0.0"
+  },
+  "devDependencies": {
+    "brittle": "^1.4.1",
+    "hyperswarm": "next",
+    "random-access-memory": "^3.1.2",
+    "standard": "^16.0.3",
+    "tmp-promise": "^3.0.2"
   },
-  "homepage": "https://github.com/mafintosh/hypercore"
+  "optionalDependencies": {
+    "fsctl": "^1.0.0"
+  }
 }

package/test/basic.js

@@ -1,502 +1,90 @@
-var create = require('./helpers/create')
-var createTrackingRam = require('./helpers/create-tracking-ram')
-var crypto = require('hypercore-crypto')
-var tape = require('tape')
-var hypercore = require('../')
-var ram = require('random-access-memory')
-var bufferAlloc = require('buffer-alloc-unsafe')
+const test = require('brittle')
+const ram = require('random-access-memory')
 
-tape('append', function (t) {
-  t.plan(8)
+const Hypercore = require('../')
+const { create } = require('./helpers')
 
-  var feed = create({ valueEncoding: 'json' })
+test('basic', async function (t) {
+  const core = await create()
+  let appends = 0
 
-  feed.append({
-    hello: 'world'
-  })
-
-  feed.append([{
-    hello: 'verden'
-  }, {
-    hello: 'welt'
-  }])
-
-  feed.flush(function () {
-    t.same(feed.length, 3, '3 blocks')
-    t.same(feed.byteLength, 54, '54 bytes')
-
-    feed.get(0, function (err, value) {
-      t.error(err, 'no error')
-      t.same(value, { hello: 'world' })
-    })
-
-    feed.get(1, function (err, value) {
-      t.error(err, 'no error')
-      t.same(value, { hello: 'verden' })
-    })
-
-    feed.get(2, function (err, value) {
-      t.error(err, 'no error')
-      t.same(value, { hello: 'welt' })
-    })
-  })
-})
-
-tape('flush', function (t) {
-  var feed = create()
-
-  feed.append('hello')
-
-  feed.flush(function (err) {
-    t.error(err, 'no error')
-    t.same(feed.length, 1, '1 block')
-    t.end()
-  })
-})
-
-tape('verify', function (t) {
-  t.plan(9)
-
-  var feed = create()
-  var evilfeed = create(feed.key, { secretKey: feed.secretKey })
-
-  feed.append('test', function (err) {
-    t.error(err, 'no error')
-
-    evilfeed.append('t\0st', function (err) {
-      t.error(err, 'no error')
-
-      feed.signature(0, function (err, sig) {
-        t.error(err, 'no error')
-        t.same(sig.index, 0, '0 signed at 0')
-
-        feed.verify(0, sig.signature, function (err, success) {
-          t.error(err, 'no error')
-          t.ok(success)
-        })
-
-        evilfeed.verify(0, sig.signature, function (err, success) {
-          t.ok(!!err)
-          t.ok(err instanceof Error)
-          t.ok(!success, 'fake verify failed')
-        })
-      })
-    })
-  })
-})
-
-tape('rootHashes', function (t) {
-  t.plan(9)
-
-  var feed = create()
-  var evilfeed = create(feed.key, { secretKey: feed.secretKey })
-
-  feed.append('test', function (err) {
-    t.error(err, 'no error')
-
-    evilfeed.append('t\0st', function (err) {
-      t.error(err, 'no error')
-
-      var result = []
+  t.is(core.length, 0)
+  t.is(core.byteLength, 0)
+  t.is(core.writable, true)
+  t.is(core.readable, true)
 
-      feed.rootHashes(0, onroots)
-      evilfeed.rootHashes(0, onroots)
-
-      function onroots (err, roots) {
-        t.error(err, 'no error')
-        t.ok(roots instanceof Array)
-        result.push(roots)
-        if (result.length < 2) return
-        t.notEqual(result[0], result[1])
-        t.equal(result[0].length, result[1].length)
-        t.notEqual(Buffer.compare(result[0][0].hash, result[1][0].hash), 0)
-      }
-    })
+  core.on('append', function () {
+    appends++
   })
-})
 
-tape('pass in secret key', function (t) {
-  var keyPair = crypto.keyPair()
-  var secretKey = keyPair.secretKey
-  var key = keyPair.publicKey
+  await core.append('hello')
+  await core.append('world')
 
-  var feed = create(key, { secretKey: secretKey })
+  t.is(core.length, 2)
+  t.is(core.byteLength, 10)
+  t.is(appends, 2)
 
-  feed.on('ready', function () {
-    t.same(feed.key, key)
-    t.same(feed.secretKey, secretKey)
-    t.ok(feed.writable)
-    t.end()
-  })
+  t.end()
 })
 
-tape('check existing key', function (t) {
-  var feed = hypercore(storage)
+test('session', async function (t) {
+  const core = await create()
 
-  feed.append('hi', function () {
-    var key = bufferAlloc(32)
-    key.fill(0)
-    var otherFeed = hypercore(storage, key)
-    otherFeed.on('error', function () {
-      t.pass('should error')
-      t.end()
-    })
-  })
+  const session = core.session()
 
-  function storage (name) {
-    if (storage[name]) return storage[name]
-    storage[name] = ram()
-    return storage[name]
-  }
+  await session.append('test')
+  t.alike(await core.get(0), Buffer.from('test'))
+  t.alike(await session.get(0), Buffer.from('test'))
+  t.end()
 })
 
-tape('create from existing keys', function (t) {
-  t.plan(3)
-
-  var storage1 = storage.bind(null, '1')
-  var storage2 = storage.bind(null, '2')
-
-  var feed = hypercore(storage1)
+test('close', async function (t) {
+  const core = await create()
+  await core.append('hello world')
 
-  feed.append('hi', function () {
-    var otherFeed = hypercore(storage2, feed.key, { secretKey: feed.secretKey })
-    var store = otherFeed._storage
-    otherFeed.ready(function () {
-      store.open({ key: feed.key }, function (err, data) {
-        t.error(err)
-        t.equals(data.key.toString('hex'), feed.key.toString('hex'))
-        t.equals(data.secretKey.toString('hex'), feed.secretKey.toString('hex'))
-      })
-    })
-  })
+  await core.close()
 
-  function storage (prefix, name) {
-    var fullname = prefix + '_' + name
-    if (storage[fullname]) return storage[fullname]
-    storage[fullname] = ram()
-    return storage[fullname]
+  try {
+    await core.get(0)
+    t.fail('core should be closed')
+  } catch {
+    t.pass('get threw correctly when core was closed')
   }
 })
 
-tape('head', function (t) {
-  t.plan(8)
-
-  var feed = create({ valueEncoding: 'json' })
-
-  feed.head(function (err, head) {
-    t.ok(!!err)
-    t.ok(err instanceof Error)
-    step2()
-  })
-
-  function step2 () {
-    feed.append({
-      hello: 'world'
-    }, function () {
-      feed.head(function (err, head) {
-        t.error(err)
-        t.same(head, { hello: 'world' })
-        step3()
-      })
-    })
-  }
-
-  function step3 () {
-    feed.append([{
-      hello: 'verden'
-    }, {
-      hello: 'welt'
-    }], function () {
-      feed.head({}, function (err, head) {
-        t.error(err)
-        t.same(head, { hello: 'welt' })
-        step4()
-      })
-    })
-  }
-
-  function step4 () {
-    feed.append('blender', function () {
-      feed.head({ valueEncoding: 'utf-8' }, function (err, head) {
-        t.error(err)
-        t.same(head, '"blender"\n')
-      })
-    })
-  }
-})
-
-tape('append, no cache', function (t) {
-  t.plan(8)
-
-  var feed = create({ valueEncoding: 'json', storageCacheSize: 0 })
-
-  feed.append({
-    hello: 'world'
-  })
-
-  feed.append([{
-    hello: 'verden'
-  }, {
-    hello: 'welt'
-  }])
-
-  feed.flush(function () {
-    t.same(feed.length, 3, '3 blocks')
-    t.same(feed.byteLength, 54, '54 bytes')
-
-    feed.get(0, function (err, value) {
-      t.error(err, 'no error')
-      t.same(value, { hello: 'world' })
-    })
-
-    feed.get(1, function (err, value) {
-      t.error(err, 'no error')
-      t.same(value, { hello: 'verden' })
-    })
-
-    feed.get(2, function (err, value) {
-      t.error(err, 'no error')
-      t.same(value, { hello: 'welt' })
-    })
-  })
-})
-
-tape('onwrite', function (t) {
-  var expected = [
-    { index: 0, data: 'hello', peer: null },
-    { index: 1, data: 'world', peer: null }
-  ]
+test('close multiple', async function (t) {
+  const core = await create()
+  await core.append('hello world')
 
-  var feed = create({
-    onwrite: function (index, data, peer, cb) {
-      t.same({ index: index, data: data.toString(), peer: peer }, expected.shift())
-      cb()
-    }
-  })
-
-  feed.append(['hello', 'world'], function (err) {
-    t.error(err, 'no error')
-    t.same(expected.length, 0)
-    t.end()
-  })
-})
+  const ev = t.test('events')
 
-tape('close, emitter and callback', function (t) {
-  t.plan(3)
-  var feed = create()
+  ev.plan(4)
 
-  feed.on('close', function () {
-    t.pass('close emitted')
-  })
+  let i = 0
 
-  feed.close(function (err) {
-    t.error(err, 'closed without error')
-    t.pass('callback invoked')
-  })
+  core.on('close', () => ev.is(i++, 0, 'on close'))
+  core.close().then(() => ev.is(i++, 1, 'first close'))
+  core.close().then(() => ev.is(i++, 2, 'second close'))
+  core.close().then(() => ev.is(i++, 3, 'third close'))
 
-  feed.close(function () {
-    t.end()
-  })
+  await ev
 })
 
-tape('close calls pending callbacks', function (t) {
-  t.plan(5)
-
-  var feed = create()
-
-  feed.createReadStream({ live: true })
-    .once('error', function (err) {
-      t.ok(err, 'read stream errors')
-    })
-    .resume()
-
-  feed.get(0, function (err) {
-    t.ok(err, 'get errors')
-  })
-
-  feed.once('close', function () {
-    t.pass('close emitted')
-  })
-
-  feed.ready(function () {
-    feed.close(function () {
-      feed.createReadStream({ live: true })
-        .once('error', function (err) {
-          t.ok(err, 'read stream still errors')
-        })
-        .resume()
-
-      feed.get(0, function (err) {
-        t.ok(err, 'get still errors')
-      })
-    })
-  })
+test('storage options', async function (t) {
+  const core = new Hypercore({ storage: ram })
+  await core.append('hello')
+  t.alike(await core.get(0), Buffer.from('hello'))
+  t.end()
 })
 
-tape('get batch', function (t) {
-  t.plan(2 * 3)
+test(
+  'allow publicKeys with different byteLength that 32, if opts.crypto were passed',
+  function (t) {
+    const key = Buffer.alloc(33).fill('a')
 
-  var feed = create({ valueEncoding: 'utf-8' })
+    const core = new Hypercore(ram, key, { crypto: {} })
 
-  feed.append(['a', 'be', 'cee', 'd'], function () {
-    feed.getBatch(0, 4, function (err, batch) {
-      t.error(err)
-      t.same(batch, ['a', 'be', 'cee', 'd'])
-    })
-    feed.getBatch(1, 3, function (err, batch) {
-      t.error(err)
-      t.same(batch, ['be', 'cee'])
-    })
-    feed.getBatch(2, 4, function (err, batch) {
-      t.error(err)
-      t.same(batch, ['cee', 'd'])
-    })
-  })
-})
-
-tape('append returns the seq', function (t) {
-  var feed = hypercore(storage)
-
-  feed.append('a', function (err, seq) {
-    t.error(err)
-    t.same(seq, 0)
-    feed.append(['b', 'c'], function (err, seq) {
-      t.error(err)
-      t.same(seq, 1)
-      feed.append(['d'], function (err, seq) {
-        t.error(err)
-        t.same(seq, 3)
-
-        var reloaded = hypercore(storage)
-        reloaded.append(['e'], function (err, seq) {
-          t.error(err)
-          t.same(seq, 4)
-          t.same(reloaded.length, 5)
-          t.end()
-        })
-      })
-    })
-  })
-
-  function storage (name) {
-    if (storage[name]) return storage[name]
-    storage[name] = ram()
-    return storage[name]
+    t.is(core.key, key)
+    t.pass('creating a core with more than 32 byteLength key did not throw')
   }
-})
-
-tape('append and createWriteStreams preserve seq', function (t) {
-  var feed = create()
-
-  var ws = feed.createWriteStream()
-
-  ws.write('a')
-  ws.write('b')
-  ws.write('c')
-  ws.end(function () {
-    t.same(feed.length, 3)
-    feed.append('d', function (err, seq) {
-      t.error(err)
-      t.same(seq, 3)
-      t.same(feed.length, 4)
-
-      var ws1 = feed.createWriteStream()
-
-      ws1.write('e')
-      ws1.write('f')
-      ws1.end(function () {
-        feed.append('g', function (err, seq) {
-          t.error(err)
-          t.same(seq, 6)
-          t.same(feed.length, 7)
-          t.end()
-        })
-      })
-    })
-  })
-})
-
-tape('closing all streams on close', function (t) {
-  var memories = {}
-  var feed = hypercore(function (filename) {
-    var memory = memories[filename]
-    if (!memory) {
-      memory = ram()
-      memories[filename] = memory
-    }
-    return memory
-  })
-  var expectedFiles = ['key', 'secret_key', 'tree', 'data', 'bitfield', 'signatures']
-  feed.ready(function () {
-    t.deepEquals(Object.keys(memories), expectedFiles, 'all files are open')
-    feed.close(function () {
-      expectedFiles.forEach(function (filename) {
-        var memory = memories[filename]
-        t.ok(memory.closed, filename + ' is closed')
-      })
-      t.end()
-    })
-  })
-})
-
-tape('writes are batched', function (t) {
-  var trackingRam = createTrackingRam()
-  var feed = hypercore(trackingRam)
-  var ws = feed.createWriteStream()
-
-  ws.write('ab')
-  ws.write('cd')
-  setImmediate(function () {
-    ws.write('ef')
-    ws.write('gh')
-    ws.end(function () {
-      t.deepEquals(trackingRam.log.data, [
-        { write: [0, Buffer.from('abcd')] },
-        { write: [4, Buffer.from('efgh')] }
-      ])
-      feed.close(function () {
-        t.end()
-      })
-    })
-  })
-})
-
-tape('cancel get', function (t) {
-  var feed = create()
-  var cancelled = false
-
-  const get = feed.get(42, function (err) {
-    t.ok(cancelled, 'was cancelled')
-    t.ok(err, 'had error')
-    t.end()
-  })
-
-  setImmediate(function () {
-    cancelled = true
-    feed.cancel(get)
-  })
-})
-
-tape('onwait', function (t) {
-  t.plan(2)
-
-  var feed = create()
-
-  feed.append('a', function () {
-    feed.get(0, {
-      onwait () {
-        t.fail('no onwait')
-      }
-    }, function () {
-      t.ok('should call cb')
-    })
-
-    feed.get(42, {
-      onwait () {
-        t.ok('should wait')
-      }
-    }, function () {
-      t.fail('no cb')
-    })
-  })
-})
+)