hypercore 10.0.0-alpha.13 → 10.0.0-alpha.17

package/README.md

@@ -63,6 +63,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
+  encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
   encryptionKey: k // optionally pass an encryption key to enable block encryption
 }
@@ -70,6 +71,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 
 You can also set valueEncoding to any [abstract-encoding](https://github.com/mafintosh/abstract-encoding) or [compact-encoding](https://github.com/compact-encoding) instance.
 
+valueEncodings will be applied to individually blocks, even if you append batches. If you want to control encoding at the batch-level, you can use the `encodeBatch` option, which is a function that takes a batch and returns a binary-encoded batch. If you provide a custom valueEncoding, it will not be applied prior to `encodeBatch`.
+
 #### `const seq = await core.append(block)`
 
 Append a block of data (or an array of blocks) to the core.
@@ -84,7 +87,7 @@ Options include
 
 ``` js
 {
-  wait: true, // wait for index to be downloaded
+  wait: true, // wait for block to be downloaded
   onwait: () => {}, // hook that is called if the get is waiting for download
   timeout: 0, // wait at max some milliseconds (0 means no timeout)
   valueEncoding: 'json' | 'utf-8' | 'binary' // defaults to the core's valueEncoding

package/index.js

@@ -14,7 +14,7 @@ const Replicator = require('./lib/replicator')
 const Extensions = require('./lib/extensions')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
-const { ReadStream } = require('./lib/streams')
+const { ReadStream, WriteStream } = require('./lib/streams')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -55,6 +55,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
 
     this.valueEncoding = null
+    this.encodeBatch = null
+
     this.key = key || null
     this.discoveryKey = null
     this.readable = true
@@ -131,7 +133,6 @@ module.exports = class Hypercore extends EventEmitter {
   _passCapabilities (o) {
     if (!this.sign) this.sign = o.sign
     this.crypto = o.crypto
-    this.opened = o.opened
     this.key = o.key
     this.discoveryKey = o.discoveryKey
     this.core = o.core
@@ -193,6 +194,9 @@ module.exports = class Hypercore extends EventEmitter {
     if (opts.valueEncoding) {
       this.valueEncoding = c.from(codecs(opts.valueEncoding))
     }
+    if (opts.encodeBatch) {
+      this.encodeBatch = opts.encodeBatch
+    }
 
     // This is a hidden option that's only used by Corestore.
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
@@ -205,7 +209,7 @@ module.exports = class Hypercore extends EventEmitter {
   async _openCapabilities (keyPair, storage, opts) {
     if (opts.from) return this._openFromExisting(opts.from, opts)
 
-    if (!this.storage) this.storage = Hypercore.defaultStorage(opts.storage || storage)
+    this.storage = Hypercore.defaultStorage(opts.storage || storage)
 
     this.core = await Core.open(this.storage, {
       keyPair,
@@ -412,6 +416,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.core.bitfield.get(index)) {
       block = await this.core.blocks.get(index)
     } else {
+      if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index)
       block = await this.replicator.requestBlock(index)
     }
@@ -424,6 +429,10 @@ module.exports = class Hypercore extends EventEmitter {
     return new ReadStream(this, opts)
   }
 
+  createWriteStream (opts) {
+    return new WriteStream(this, opts)
+  }
+
   download (range) {
     const linear = !!(range && range.linear)
 
@@ -481,10 +490,13 @@ module.exports = class Hypercore extends EventEmitter {
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
     const preappend = this.encryption && this._preappend
-    const buffers = new Array(blocks.length)
 
-    for (let i = 0; i < blocks.length; i++) {
-      buffers[i] = this._encode(this.valueEncoding, blocks[i])
+    const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)
+
+    if (this.encodeBatch === null) {
+      for (let i = 0; i < blocks.length; i++) {
+        buffers[i] = this._encode(this.valueEncoding, blocks[i])
+      }
     }
 
     return await this.core.append(buffers, this.sign, { preappend })

package/lib/core.js

@@ -1,4 +1,5 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
@@ -90,7 +91,7 @@ module.exports = class Core {
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
       throw new Error('Another hypercore is stored here')
     }
 
@@ -176,7 +177,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }

package/lib/merkle-tree.js

@@ -215,7 +215,7 @@ class ReorgBatch extends MerkleTreeBatch {
     const nodes = []
     const root = verifyBlock(proof, this.tree.crypto, nodes)
 
-    if (root === null || !root.hash.equals(this.diff.hash)) return false
+    if (root === null || !b4a.equals(root.hash, this.diff.hash)) return false
 
     this.nodes.push(...nodes)
     return this._update(nodes)
@@ -233,7 +233,7 @@ class ReorgBatch extends MerkleTreeBatch {
       if (!left) break
 
       const existing = await this.tree.get(left.index, false)
-      if (!existing || !existing.hash.equals(left.hash)) {
+      if (!existing || !b4a.equals(existing.hash, left.hash)) {
         diff = left
       } else {
         diff = n.get(ite.sibling())
@@ -355,7 +355,7 @@ module.exports = class MerkleTree {
   }
 
   addNode (node) {
-    if (node.size === 0 && node.hash.equals(BLANK_HASH)) node = blankNode(node.index)
+    if (node.size === 0 && b4a.equals(node.hash, BLANK_HASH)) node = blankNode(node.index)
     this.unflushed.set(node.index, node)
   }
 
@@ -379,6 +379,17 @@ module.exports = class MerkleTree {
     return this.signature !== null && this.crypto.verify(this.signable(), this.signature, key)
   }
 
+  getRoots (length) {
+    const indexes = flat.fullRoots(2 * length)
+    const roots = new Array(indexes.length)
+
+    for (let i = 0; i < indexes.length; i++) {
+      roots[i] = this.get(indexes[i], true)
+    }
+
+    return Promise.all(roots)
+  }
+
   get (index, error = true) {
     let node = this.unflushed.get(index)
 
@@ -534,7 +545,7 @@ module.exports = class MerkleTree {
 
     for (const root of batch.roots) {
       const existing = await this.get(root.index, false)
-      if (existing && existing.hash.equals(root.hash)) continue
+      if (existing && b4a.equals(existing.hash, root.hash)) continue
       batch._updateDiffRoot(root)
       break
     }
@@ -562,7 +573,7 @@ module.exports = class MerkleTree {
 
     if (unverified) {
       const verified = await this.get(unverified.index)
-      if (!verified.hash.equals(unverified.hash)) {
+      if (!b4a.equals(verified.hash, unverified.hash)) {
         throw new Error('Invalid checksum at node ' + unverified.index)
       }
     }
@@ -707,7 +718,7 @@ module.exports = class MerkleTree {
     await new Promise((resolve, reject) => {
       storage.read(0, OLD_TREE.length, (err, buf) => {
         if (err) return resolve()
-        if (buf.equals(OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
+        if (b4a.equals(buf, OLD_TREE)) return reject(new Error('Storage contains an incompatible merkle tree'))
         resolve()
       })
     })

package/lib/streams.js

@@ -1,4 +1,4 @@
-const { Readable } = require('streamx')
+const { Writable, Readable } = require('streamx')
 
 class ReadStream extends Readable {
   constructor (core, opts = {}) {
@@ -37,3 +37,20 @@ class ReadStream extends Readable {
 }
 
 exports.ReadStream = ReadStream
+
+class WriteStream extends Writable {
+  constructor (core) {
+    super()
+    this.core = core
+  }
+
+  _writev (batch, cb) {
+    this._writevP(batch).then(cb, cb)
+  }
+
+  async _writevP (batch) {
+    await this.core.append(batch)
+  }
+}
+
+exports.WriteStream = WriteStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.13",
+  "version": "10.0.0-alpha.17",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -27,13 +27,18 @@
     "url": "https://github.com/hypercore-protocol/hypercore/issues"
   },
   "homepage": "https://github.com/hypercore-protocol/hypercore#readme",
+  "files": [
+    "index.js",
+    "lib/**.js"
+  ],
   "dependencies": {
     "@hyperswarm/secret-stream": "^5.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
-    "codecs": "^2.2.0",
+    "codecs": "^3.0.0",
     "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",
+    "events": "^3.3.0",
     "flat-tree": "^1.9.0",
     "hypercore-crypto": "^3.1.0",
     "is-options": "^1.0.1",

package/.github/workflows/test-node.yml

@@ -1,23 +0,0 @@
-name: Build Status
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-jobs:
-  build:
-    strategy:
-      matrix:
-        node-version: [lts/*]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm install
-    - run: npm test

package/UPGRADE.md

@@ -1,15 +0,0 @@
-# Upgrade Notes
-
-Notes for downstream developers who are upgrading their modules to new, breaking versions of hypercore.
-
-## 10.0.0
-
-- All number encodings are now LE
-- Introduces an "oplog" to atomically track changes locally
-- Updated merkle format that only requires a single signature (stored in the oplog)
-
-## 9.0.0
-
-- The format of signatures [has been changed](https://github.com/mafintosh/hypercore/issues/260). This is backwards-compatible (v9 can read v8 signatures), but forward-incompatible (v8 cannot read v9 signatures). If a v8 peer replicates with a v9 peer, it will emit a "REMOTE SIGNTURE INVALID" error on the replication stream.
-- The encryption ([NOISE](https://github.com/emilbayes/noise-protocol)) handshake has been changed in an backwards- and forwards-incompatible way. v8 peers can not handshake with v9 peers, and vice-versa. A NOISE-related error is emitted on the replication stream.
-- There is no way (yet) to detect whether a peer is running an incompatible version of hypercore at the replication level. One workaround for downstream developers is to include their own application-level handshake before piping to the replication stream, to communicate a "app protocol version" (maybe "v8" and "v9") and abort the connection if the peer is running an incompatible version.