hypercore 10.0.0-alpha.14 → 10.0.0-alpha.18

package/README.md

@@ -63,6 +63,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
+  encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
   encryptionKey: k // optionally pass an encryption key to enable block encryption
 }
@@ -70,6 +71,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 
 You can also set valueEncoding to any [abstract-encoding](https://github.com/mafintosh/abstract-encoding) or [compact-encoding](https://github.com/compact-encoding) instance.
 
+valueEncodings will be applied to individually blocks, even if you append batches. If you want to control encoding at the batch-level, you can use the `encodeBatch` option, which is a function that takes a batch and returns a binary-encoded batch. If you provide a custom valueEncoding, it will not be applied prior to `encodeBatch`.
+
 #### `const seq = await core.append(block)`
 
 Append a block of data (or an array of blocks) to the core.
@@ -84,7 +87,7 @@ Options include
 
 ``` js
 {
-  wait: true, // wait for index to be downloaded
+  wait: true, // wait for block to be downloaded
   onwait: () => {}, // hook that is called if the get is waiting for download
   timeout: 0, // wait at max some milliseconds (0 means no timeout)
   valueEncoding: 'json' | 'utf-8' | 'binary' // defaults to the core's valueEncoding

package/index.js

@@ -14,7 +14,7 @@ const Replicator = require('./lib/replicator')
 const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
-const { ReadStream } = require('./lib/streams')
+const { ReadStream, WriteStream } = require('./lib/streams')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -55,6 +55,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
+    this.encodeBatch = null
+
     this.key = key || null
     this.discoveryKey = null
     this.readable = true
@@ -90,9 +92,27 @@ module.exports = class Hypercore extends EventEmitter {
       indent + ')'
   }
 
-  static createProtocolStream (isInitiator, opts) {
-    const noiseStream = new NoiseSecretStream(isInitiator, null, opts)
-    return noiseStream.rawStream
+  static createProtocolStream (isInitiator, opts = {}) {
+    let outerStream = isStream(isInitiator)
+      ? isInitiator
+      : opts.stream
+    let noiseStream = null
+
+    if (outerStream) {
+      noiseStream = outerStream.noiseStream
+    } else {
+      noiseStream = new NoiseSecretStream(isInitiator, null, opts)
+      outerStream = noiseStream.rawStream
+    }
+    if (!noiseStream) throw new Error('Invalid stream')
+
+    if (!noiseStream.userData) {
+      const protocol = Replicator.createProtocol(noiseStream)
+      noiseStream.userData = protocol
+      noiseStream.on('error', noop) // All noise errors already propagate through outerStream
+    }
+
+    return outerStream
   }
 
   static defaultStorage (storage, opts = {}) {
@@ -192,6 +212,9 @@ module.exports = class Hypercore extends EventEmitter {
     if (opts.valueEncoding) {
       this.valueEncoding = c.from(codecs(opts.valueEncoding))
     }
+    if (opts.encodeBatch) {
+      this.encodeBatch = opts.encodeBatch
+    }
 
     // This is a hidden option that's only used by Corestore.
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
@@ -263,33 +286,17 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   replicate (isInitiator, opts = {}) {
-    let outerStream = isStream(isInitiator)
-      ? isInitiator
-      : opts.stream
-    let noiseStream = null
-
-    if (outerStream) {
-      noiseStream = outerStream.noiseStream
-    } else {
-      outerStream = Hypercore.createProtocolStream(isInitiator, opts)
-      noiseStream = outerStream.noiseStream
-    }
-    if (!noiseStream) throw new Error('Invalid stream passed to replicate')
-
-    if (!noiseStream.userData) {
-      const protocol = Replicator.createProtocol(noiseStream)
-      noiseStream.userData = protocol
-      noiseStream.on('error', noop) // All noise errors already propagate through outerStream
-    }
-
+    const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
+    const noiseStream = protocolStream.noiseStream
     const protocol = noiseStream.userData
+
     if (this.opened) {
       this.replicator.joinProtocol(protocol, this.key, this.discoveryKey)
     } else {
       this.opening.then(() => this.replicator.joinProtocol(protocol, this.key, this.discoveryKey), protocol.destroy.bind(protocol))
     }
 
-    return outerStream
+    return protocolStream
   }
 
   get length () {
@@ -411,6 +418,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.core.bitfield.get(index)) {
       block = await this.core.blocks.get(index)
     } else {
+      if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index)
       block = await this.replicator.requestBlock(index)
     }
@@ -423,6 +431,10 @@ module.exports = class Hypercore extends EventEmitter {
     return new ReadStream(this, opts)
   }
 
+  createWriteStream (opts) {
+    return new WriteStream(this, opts)
+  }
+
   download (range) {
     const linear = !!(range && range.linear)
 
@@ -480,10 +492,13 @@ module.exports = class Hypercore extends EventEmitter {
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
     const preappend = this.encryption && this._preappend
-    const buffers = new Array(blocks.length)
 
-    for (let i = 0; i < blocks.length; i++) {
-      buffers[i] = this._encode(this.valueEncoding, blocks[i])
+    const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
+
+    if (this.encodeBatch === null) {
+      for (let i = 0; i < blocks.length; i++) {
+        buffers[i] = this._encode(this.valueEncoding, blocks[i])
+      }
     }
 
     return await this.core.append(buffers, this.sign, { preappend })

package/lib/core.js

@@ -1,4 +1,5 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
@@ -90,7 +91,7 @@ module.exports = class Core {
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
       throw new Error('Another hypercore is stored here')
     }
 
@@ -176,7 +177,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }

package/lib/merkle-tree.js

@@ -215,7 +215,7 @@ class ReorgBatch extends MerkleTreeBatch {
     const nodes = []
     const root = verifyBlock(proof, this.tree.crypto, nodes)
 
-    if (root === null || !root.hash.equals(this.diff.hash)) return false
+    if (root === null || !b4a.equals(root.hash, this.diff.hash)) return false
 
     this.nodes.push(...nodes)
     return this._update(nodes)
@@ -233,7 +233,7 @@ class ReorgBatch extends MerkleTreeBatch {
       if (!left) break
 
       const existing = await this.tree.get(left.index, false)
-      if (!existing || !existing.hash.equals(left.hash)) {
+      if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
         diff = n.get(ite.sibling())
@@ -355,7 +355,7 @@ module.exports = class MerkleTree {
   }
 
   addNode (node) {
-    if (node.size === 0 && node.hash.equals(BLANK_HASH)) node = blankNode(node.index)
+    if (node.size === 0 && b4a.equals(node.hash, BLANK_HASH)) node = blankNode(node.index)
     this.unflushed.set(node.index, node)
   }
 
@@ -379,6 +379,17 @@ module.exports = class MerkleTree {
     return this.signature !== null && this.crypto.verify(this.signable(), this.signature, key)
   }
 
+  getRoots (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], true)
+    }
+
+    return Promise.all(roots)
+  }
+
   get (index, error = true) {
     let node = this.unflushed.get(index)
 
@@ -534,7 +545,7 @@ module.exports = class MerkleTree {
 
     for (const root of batch.roots) {
       const existing = await this.get(root.index, false)
-      if (existing && existing.hash.equals(root.hash)) continue
+      if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
     }
@@ -562,7 +573,7 @@ module.exports = class MerkleTree {
 
     if (unverified) {
       const verified = await this.get(unverified.index)
-      if (!verified.hash.equals(unverified.hash)) {
+      if (!b4a.equals(verified.hash, unverified.hash)) {
         throw new Error('Invalid checksum at node ' + unverified.index)
       }
     }
@@ -707,7 +718,7 @@ module.exports = class MerkleTree {
     await new Promise((resolve, reject) => {
       storage.read(0, OLD_TREE.length, (err, buf) => {
         if (err) return resolve()
-        if (buf.equals(OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
+        if (b4a.equals(buf, OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
         resolve()
       })
     })

package/lib/protocol.js

@@ -39,7 +39,7 @@ class Extension {
   _sendAlias (message, alias) {
     if (this.destroyed) return
 
-    if (this._remoteAliases) {
+    if (this.remoteSupports) {
       return this.protocol.send(this.type, this.encoding, alias, message)
     }
 

package/lib/streams.js

@@ -1,4 +1,4 @@
-const { Readable } = require('streamx')
+const { Writable, Readable } = require('streamx')
 
 class ReadStream extends Readable {
   constructor (core, opts = {}) {
@@ -37,3 +37,20 @@ class ReadStream extends Readable {
 }
 
 exports.ReadStream = ReadStream
+
+class WriteStream extends Writable {
+  constructor (core) {
+    super()
+    this.core = core
+  }
+
+  _writev (batch, cb) {
+    this._writevP(batch).then(cb, cb)
+  }
+
+  async _writevP (batch) {
+    await this.core.append(batch)
+  }
+}
+
+exports.WriteStream = WriteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.14",
+  "version": "10.0.0-alpha.18",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -35,9 +35,10 @@
     "@hyperswarm/secret-stream": "^5.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
-    "codecs": "^2.2.0",
+    "codecs": "^3.0.0",
     "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",
+    "events": "^3.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.1.0",
     "is-options": "^1.0.1",