hypercore 10.0.0-alpha.12 → 10.0.0-alpha.16

package/README.md

@@ -63,6 +63,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
+  encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
   encryptionKey: k // optionally pass an encryption key to enable block encryption
 }
@@ -70,6 +71,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 
 You can also set valueEncoding to any [abstract-encoding](https://github.com/mafintosh/abstract-encoding) or [compact-encoding](https://github.com/compact-encoding) instance.
 
+valueEncodings will be applied to individually blocks, even if you append batches. If you want to control encoding at the batch-level, you can use the `encodeBatch` option, which is a function that takes a batch and returns a binary-encoded batch. If you provide a custom valueEncoding, it will not be applied prior to `encodeBatch`.
+
 #### `const seq = await core.append(block)`
 
 Append a block of data (or an array of blocks) to the core.

package/index.js

@@ -14,7 +14,7 @@ const Replicator = require('./lib/replicator')
 const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
-const { ReadStream } = require('./lib/streams')
+const { ReadStream, WriteStream } = require('./lib/streams')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -55,6 +55,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
+    this.encodeBatch = null
+
     this.key = key || null
     this.discoveryKey = null
     this.readable = true
@@ -66,7 +68,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.autoClose = !!opts.autoClose
 
     this.closing = null
-    this.opening = opts._opening || this._open(key, storage, opts)
+    this.opening = this._openSession(key, storage, opts)
     this.opening.catch(noop)
 
     this._preappend = preappend.bind(this)
@@ -115,31 +117,22 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     const Clz = opts.class || Hypercore
-    const keyPair = opts.keyPair && opts.keyPair.secretKey && { ...opts.keyPair }
-
-    // This only works if the hypercore was fully loaded,
-    // but we only do this to validate the keypair to help catch bugs so yolo
-    if (this.key && keyPair) keyPair.publicKey = this.key
-
     const s = new Clz(this.storage, this.key, {
       ...opts,
-      sign: opts.sign || (keyPair && keyPair.secretKey && Core.createSigner(this.crypto, keyPair)) || this.sign,
-      valueEncoding: this.valueEncoding,
       extensions: this.extensions,
       _opening: this.opening,
       _sessions: this.sessions
     })
 
-    s._initSession(this)
+    s._passCapabilities(this)
     this.sessions.push(s)
 
     return s
   }
 
-  _initSession (o) {
+  _passCapabilities (o) {
     if (!this.sign) this.sign = o.sign
     this.crypto = o.crypto
-    this.opened = o.opened
     this.key = o.key
     this.discoveryKey = o.discoveryKey
     this.core = o.core
@@ -149,6 +142,101 @@ module.exports = class Hypercore extends EventEmitter {
     this.autoClose = o.autoClose
   }
 
+  async _openFromExisting (from, opts) {
+    await from.opening
+
+    for (const [name, ext] of this.extensions) {
+      from.extensions.register(name, null, ext)
+    }
+
+    this._passCapabilities(from)
+    this.extensions = from.extensions
+    this.sessions = from.sessions
+    this.storage = from.storage
+
+    this.sessions.push(this)
+  }
+
+  async _openSession (key, storage, opts) {
+    const isFirst = !opts._opening
+
+    if (!isFirst) await opts._opening
+    if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
+
+    const keyPair = (key && opts.keyPair)
+      ? { ...opts.keyPair, publicKey: key }
+      : key
+        ? { publicKey: key, secretKey: null }
+        : opts.keyPair
+
+    // This only works if the hypercore was fully loaded,
+    // but we only do this to validate the keypair to help catch bugs so yolo
+    if (this.key && keyPair) keyPair.publicKey = this.key
+
+    if (opts.sign) {
+      this.sign = opts.sign
+    } else if (keyPair && keyPair.secretKey) {
+      this.sign = Core.createSigner(this.crypto, keyPair)
+    }
+
+    if (isFirst) {
+      await this._openCapabilities(keyPair, storage, opts)
+      // Only the root session should pass capabilities to other sessions.
+      for (let i = 0; i < this.sessions.length; i++) {
+        const s = this.sessions[i]
+        if (s !== this) s._passCapabilities(this)
+      }
+    }
+
+    if (!this.sign) this.sign = this.core.defaultSign
+    this.writable = !!this.sign
+
+    if (opts.valueEncoding) {
+      this.valueEncoding = c.from(codecs(opts.valueEncoding))
+    }
+    if (opts.encodeBatch) {
+      this.encodeBatch = opts.encodeBatch
+    }
+
+    // This is a hidden option that's only used by Corestore.
+    // It's required so that corestore can load a name from userData before 'ready' is emitted.
+    if (opts._preready) await opts._preready(this)
+
+    this.opened = true
+    this.emit('ready')
+  }
+
+  async _openCapabilities (keyPair, storage, opts) {
+    if (opts.from) return this._openFromExisting(opts.from, opts)
+
+    this.storage = Hypercore.defaultStorage(opts.storage || storage)
+
+    this.core = await Core.open(this.storage, {
+      keyPair,
+      crypto: this.crypto,
+      onupdate: this._oncoreupdate.bind(this)
+    })
+
+    if (opts.userData) {
+      for (const [key, value] of Object.entries(opts.userData)) {
+        await this.core.userData(key, value)
+      }
+    }
+
+    this.replicator = new Replicator(this.core, {
+      onupdate: this._onpeerupdate.bind(this)
+    })
+
+    this.discoveryKey = this.crypto.discoveryKey(this.core.header.signer.publicKey)
+    this.key = this.core.header.signer.publicKey
+
+    if (!this.encryption && opts.encryptionKey) {
+      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
+    }
+
+    this.extensions.attach(this.replicator)
+  }
+
   close () {
     if (this.closing) return this.closing
     this.closing = this._close()
@@ -237,71 +325,6 @@ module.exports = class Hypercore extends EventEmitter {
     return this.opening
   }
 
-  async _open (key, storage, opts) {
-    if (opts.preload) opts = { ...opts, ...(await opts.preload()) }
-
-    this.valueEncoding = opts.valueEncoding ? c.from(codecs(opts.valueEncoding)) : null
-
-    const keyPair = (key && opts.keyPair)
-      ? { ...opts.keyPair, publicKey: key }
-      : key
-        ? { publicKey: key, secretKey: null }
-        : opts.keyPair
-
-    if (opts.from) {
-      const from = opts.from
-      await from.opening
-      for (const [name, ext] of this.extensions) from.extensions.register(name, null, ext)
-      this._initSession(from)
-      this.extensions = from.extensions
-      this.sessions = from.sessions
-      this.storage = from.storage
-      if (!this.sign) this.sign = opts.sign || ((keyPair && keyPair.secretKey) ? Core.createSigner(this.crypto, keyPair) : null)
-      this.writable = !!this.sign
-      this.sessions.push(this)
-      return
-    }
-
-    if (!this.storage) this.storage = Hypercore.defaultStorage(opts.storage || storage)
-
-    this.core = await Core.open(this.storage, {
-      keyPair,
-      crypto: this.crypto,
-      onupdate: this._oncoreupdate.bind(this)
-    })
-
-    if (opts.userData) {
-      for (const [key, value] of Object.entries(opts.userData)) {
-        await this.core.userData(key, value)
-      }
-    }
-
-    this.replicator = new Replicator(this.core, {
-      onupdate: this._onpeerupdate.bind(this)
-    })
-
-    if (!this.sign) this.sign = opts.sign || this.core.defaultSign
-
-    this.discoveryKey = this.crypto.discoveryKey(this.core.header.signer.publicKey)
-    this.key = this.core.header.signer.publicKey
-    this.writable = !!this.sign
-
-    if (!this.encryption && opts.encryptionKey) {
-      this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
-    }
-
-    this.extensions.attach(this.replicator)
-    this.opened = true
-
-    if (opts.postload) await opts.postload(this)
-
-    for (let i = 0; i < this.sessions.length; i++) {
-      const s = this.sessions[i]
-      if (s !== this) s._initSession(this)
-      s.emit('ready')
-    }
-  }
-
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
       for (let i = 0; i < this.sessions.length; i++) {
@@ -405,6 +428,10 @@ module.exports = class Hypercore extends EventEmitter {
     return new ReadStream(this, opts)
   }
 
+  createWriteStream (opts) {
+    return new WriteStream(this, opts)
+  }
+
   download (range) {
     const linear = !!(range && range.linear)
 
@@ -462,10 +489,13 @@ module.exports = class Hypercore extends EventEmitter {
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
     const preappend = this.encryption && this._preappend
-    const buffers = new Array(blocks.length)
 
-    for (let i = 0; i < blocks.length; i++) {
-      buffers[i] = this._encode(this.valueEncoding, blocks[i])
+    const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
+
+    if (this.encodeBatch === null) {
+      for (let i = 0; i < blocks.length; i++) {
+        buffers[i] = this._encode(this.valueEncoding, blocks[i])
+      }
     }
 
     return await this.core.append(buffers, this.sign, { preappend })

package/lib/core.js

@@ -1,4 +1,5 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
@@ -90,7 +91,7 @@ module.exports = class Core {
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
       throw new Error('Another hypercore is stored here')
     }
 
@@ -176,7 +177,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }

package/lib/merkle-tree.js

@@ -215,7 +215,7 @@ class ReorgBatch extends MerkleTreeBatch {
     const nodes = []
     const root = verifyBlock(proof, this.tree.crypto, nodes)
 
-    if (root === null || !root.hash.equals(this.diff.hash)) return false
+    if (root === null || !b4a.equals(root.hash, this.diff.hash)) return false
 
     this.nodes.push(...nodes)
     return this._update(nodes)
@@ -233,7 +233,7 @@ class ReorgBatch extends MerkleTreeBatch {
       if (!left) break
 
       const existing = await this.tree.get(left.index, false)
-      if (!existing || !existing.hash.equals(left.hash)) {
+      if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
         diff = n.get(ite.sibling())
@@ -355,7 +355,7 @@ module.exports = class MerkleTree {
   }
 
   addNode (node) {
-    if (node.size === 0 && node.hash.equals(BLANK_HASH)) node = blankNode(node.index)
+    if (node.size === 0 && b4a.equals(node.hash, BLANK_HASH)) node = blankNode(node.index)
     this.unflushed.set(node.index, node)
   }
 
@@ -379,6 +379,17 @@ module.exports = class MerkleTree {
     return this.signature !== null && this.crypto.verify(this.signable(), this.signature, key)
   }
 
+  getRoots (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], true)
+    }
+
+    return Promise.all(roots)
+  }
+
   get (index, error = true) {
     let node = this.unflushed.get(index)
 
@@ -534,7 +545,7 @@ module.exports = class MerkleTree {
 
     for (const root of batch.roots) {
       const existing = await this.get(root.index, false)
-      if (existing && existing.hash.equals(root.hash)) continue
+      if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
     }
@@ -562,7 +573,7 @@ module.exports = class MerkleTree {
 
     if (unverified) {
       const verified = await this.get(unverified.index)
-      if (!verified.hash.equals(unverified.hash)) {
+      if (!b4a.equals(verified.hash, unverified.hash)) {
         throw new Error('Invalid checksum at node ' + unverified.index)
       }
     }
@@ -707,7 +718,7 @@ module.exports = class MerkleTree {
     await new Promise((resolve, reject) => {
       storage.read(0, OLD_TREE.length, (err, buf) => {
         if (err) return resolve()
-        if (buf.equals(OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
+        if (b4a.equals(buf, OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
         resolve()
       })
     })

package/lib/streams.js

@@ -1,4 +1,4 @@
-const { Readable } = require('streamx')
+const { Writable, Readable } = require('streamx')
 
 class ReadStream extends Readable {
   constructor (core, opts = {}) {
@@ -37,3 +37,20 @@ class ReadStream extends Readable {
 }
 
 exports.ReadStream = ReadStream
+
+class WriteStream extends Writable {
+  constructor (core) {
+    super()
+    this.core = core
+  }
+
+  _writev (batch, cb) {
+    this._writevP(batch).then(cb, cb)
+  }
+
+  async _writevP (batch) {
+    await this.core.append(batch)
+  }
+}
+
+exports.WriteStream = WriteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.12",
+  "version": "10.0.0-alpha.16",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -27,15 +27,20 @@
     "url": "https://github.com/hypercore-protocol/hypercore/issues"
   },
   "homepage": "https://github.com/hypercore-protocol/hypercore#readme",
+  "files": [
+    "index.js",
+    "lib/**.js"
+  ],
   "dependencies": {
     "@hyperswarm/secret-stream": "^5.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
-    "codecs": "^2.2.0",
+    "codecs": "^3.0.0",
     "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",
+    "events": "^3.3.0",
     "flat-tree": "^1.9.0",
-    "hypercore-crypto": "^3.0.0",
+    "hypercore-crypto": "^3.1.0",
     "is-options": "^1.0.1",
     "random-access-file": "^2.1.4",
     "random-array-iterator": "^1.0.0",

package/.github/workflows/test-node.yml

@@ -1,23 +0,0 @@
-name: Build Status
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-jobs:
-  build:
-    strategy:
-      matrix:
-        node-version: [lts/*]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm test

package/UPGRADE.md

@@ -1,9 +0,0 @@
-# Upgrade Notes
-
-Notes for downstream developers who are upgrading their modules to new, breaking versions of hypercore.
-
-## 9.0.0
-
-- The format of signatures [has been changed](https://github.com/mafintosh/hypercore/issues/260). This is backwards-compatible (v9 can read v8 signatures), but forward-incompatible (v8 cannot read v9 signatures). If a v8 peer replicates with a v9 peer, it will emit a "REMOTE SIGNTURE INVALID" error on the replication stream.
-- The encryption ([NOISE](https://github.com/emilbayes/noise-protocol)) handshake has been changed in an backwards- and forwards-incompatible way. v8 peers can not handshake with v9 peers, and vice-versa. A NOISE-related error is emitted on the replication stream.
-- There is no way (yet) to detect whether a peer is running an incompatible version of hypercore at the replication level. One workaround for downstream developers is to include their own application-level handshake before piping to the replication stream, to communicate a "app protocol version" (maybe "v8" and "v9") and abort the connection if the peer is running an incompatible version.