hypercore 10.22.2 → 10.23.0

package/index.js

@@ -15,6 +15,7 @@ const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const Batch = require('./lib/batch')
+const { manifestHash, defaultSignerManifest, createVerifier, createManifest, isCompat } = require('./lib/manifest')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const {
   BAD_ARGUMENT,
@@ -69,7 +70,7 @@ module.exports = class Hypercore extends EventEmitter {
 
     this.id = null
     this.key = key || null
-    this.keyPair = null
+    this.keyPair = opts.keyPair || null
     this.readable = true
     this.writable = false
     this.opened = false
@@ -77,22 +78,23 @@ module.exports = class Hypercore extends EventEmitter {
     this.snapshotted = !!opts.snapshot
     this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
-    this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
     this.onwait = opts.onwait || null
     this.wait = opts.wait !== false
     this.timeout = opts.timeout || 0
-    this._clone = opts.clone || null
-    this._readonly = opts.writable === false
-
     this.closing = null
-    this.opening = this._openSession(key, storage, opts)
-    this.opening.catch(noop)
+    this.opening = null
 
+    this._manifest = opts.manifest || null
+    this._clone = opts.clone || null
+    this._readonly = opts.writable === false
     this._preappend = preappend.bind(this)
     this._snapshot = null
     this._batch = opts._batch || null
     this._findingPeers = 0
+
+    this.opening = this._openSession(key, storage, opts)
+    this.opening.catch(noop)
   }
 
   [inspect] (depth, opts) {
@@ -139,6 +141,14 @@ module.exports = class Hypercore extends EventEmitter {
       indent + ')'
   }
 
+  static key (manifest, { compat } = {}) {
+    return compat ? manifest.signer.publicKey : manifestHash(manifest)
+  }
+
+  static discoveryKey (key) {
+    return hypercoreCrypto.discoveryKey(key)
+  }
+
   static getProtocolMuxer (stream) {
     return stream.noiseStream.userData
   }
@@ -245,20 +255,19 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   setKeyPair (keyPair) {
-    this.auth = Core.createAuth(this.crypto, { keyPair })
-    this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+    this.keyPair = keyPair
+    this.writable = this._isWritable()
   }
 
   _passCapabilities (o) {
-    if (!this.auth) this.auth = o.auth
-
+    if (!this.keyPair) this.keyPair = o.keyPair
     this.crypto = o.crypto
     this.id = o.id
     this.key = o.key
     this.core = o.core
     this.replicator = o.replicator
     this.encryption = o.encryption
-    this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+    this.writable = this._isWritable()
     this.autoClose = o.autoClose
 
     if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
@@ -288,14 +297,6 @@ module.exports = class Hypercore extends EventEmitter {
     if (!isFirst) await opts._opening
     if (opts.preload) opts = { ...opts, ...(await this._retryPreload(opts.preload)) }
 
-    const keyPair = opts.keyPair
-
-    if (opts.auth) {
-      this.auth = opts.auth
-    } else if (keyPair && keyPair.secretKey) {
-      this.setKeyPair(keyPair)
-    }
-
     if (isFirst) {
       await this._openCapabilities(key, storage, opts)
       // Only the root session should pass capabilities to other sessions.
@@ -306,15 +307,14 @@ module.exports = class Hypercore extends EventEmitter {
 
       // copy state over
       if (this._clone) {
-        const { from, upgrade } = this._clone
+        const { from, signature } = this._clone
         await from.opening
-        await this.core.copyFrom(from.core, upgrade)
+        await this.core.copyFrom(from.core, signature)
         this._clone = null
       }
     }
 
-    if (!this.auth) this.auth = this.core.defaultAuth
-    this.writable = !this._readonly && !!this.auth && !!this.auth.sign
+    this.writable = this._isWritable()
 
     if (opts.valueEncoding) {
       this.valueEncoding = c.from(opts.valueEncoding)
@@ -353,7 +353,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.storage = Hypercore.defaultStorage(opts.storage || storage, { unlocked, writable: !unlocked })
 
     this.core = await Core.open(this.storage, {
-      compat: opts.compat !== false, // default to true for now
+      compat: opts.compat,
       force: opts.force,
       createIfMissing: opts.createIfMissing,
       readonly: unlocked,
@@ -362,7 +362,7 @@ module.exports = class Hypercore extends EventEmitter {
       keyPair: opts.keyPair,
       crypto: this.crypto,
       legacy: opts.legacy,
-      auth: opts.auth,
+      manifest: opts.manifest,
       onupdate: this._oncoreupdate.bind(this),
       onconflict: this._oncoreconflict.bind(this)
     })
@@ -417,6 +417,10 @@ module.exports = class Hypercore extends EventEmitter {
     return prev.length !== next.length || prev.fork !== next.fork
   }
 
+  _isWritable () {
+    return !this._readonly && !!(this.keyPair && this.keyPair.secretKey)
+  }
+
   close (err) {
     if (this.closing) return this.closing
     this.closing = this._close(err || null)
@@ -465,23 +469,22 @@ module.exports = class Hypercore extends EventEmitter {
     this.emit('close', true)
   }
 
-  clone (storage, opts = {}) {
+  clone (keyPair, storage, opts = {}) {
     // TODO: current limitation is no forking
     if ((opts.fork && opts.fork !== 0) || this.fork !== 0) {
       throw BAD_ARGUMENT('Cannot clone a fork')
     }
 
-    const key = opts.key === undefined ? opts.keyPair ? null : this.key : opts.key
-    const keyPair = (opts.auth || opts.keyPair === undefined) ? null : opts.keyPair
+    const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
+    const key = opts.key || (opts.compat !== false ? manifest.signer.publicKey : manifestHash(manifest))
 
-    let auth = this.core.defaultAuth
-    if (opts.auth) {
-      auth = opts.auth
-    } else if (keyPair && keyPair.secretKey) {
-      auth = Core.createAuth(this.crypto, { keyPair, manifest: { signer: { publicKey: keyPair.publicKey } } })
+    if (b4a.equals(key, this.key)) {
+      throw BAD_ARGUMENT('Clone cannot share verification information')
     }
 
-    const upgrade = opts.upgrade === undefined ? null : opts.upgrade
+    const signature = opts.signature === undefined
+      ? createVerifier(createManifest(manifest), { compat: isCompat(key, manifest) }).sign(this.core.tree.batch(), keyPair)
+      : opts.signature
 
     const sparse = opts.sparse === false ? false : this.sparse
     const wait = opts.wait === false ? false : this.wait
@@ -497,11 +500,11 @@ module.exports = class Hypercore extends EventEmitter {
       wait,
       onwait,
       timeout,
-      auth,
+      manifest,
       overwrite: true,
       clone: {
         from: this,
-        upgrade
+        signature
       }
     })
   }
@@ -542,6 +545,10 @@ module.exports = class Hypercore extends EventEmitter {
     return this.replicator === null ? null : this.replicator.discoveryKey
   }
 
+  get manifest () {
+    return this._manifest || (this.core === null ? null : this.core.header.manifest)
+  }
+
   get length () {
     if (this._snapshot) return this._snapshot.length
     if (this.core === null) return 0
@@ -633,6 +640,13 @@ module.exports = class Hypercore extends EventEmitter {
         this.replicator.onupgrade()
       }
 
+      if (status & 0b10000) {
+        for (let i = 0; i < this.sessions.length; i++) {
+          const s = this.sessions[i]
+          if (!s._manifest) s.emit('manifest')
+        }
+      }
+
       for (let i = 0; i < this.sessions.length; i++) {
         const s = this.sessions[i]
 
@@ -930,24 +944,36 @@ module.exports = class Hypercore extends EventEmitter {
     // Do nothing for now
   }
 
-  async truncate (newLength = 0, fork = -1) {
-    if (this._batch) throw BATCH_UNFLUSHED()
-
+  async truncate (newLength = 0, opts = {}) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
-    if (fork === -1) fork = this.core.tree.fork + 1
-    await this.core.truncate(newLength, fork, this.auth)
+    const {
+      fork = this.core.tree.fork + 1,
+      force = false,
+      keyPair = this.keyPair,
+      signature = null
+    } = typeof opts === 'number' ? { fork: opts } : opts
+
+    const writable = !this._readonly && !!(signature || (keyPair && keyPair.secretKey))
+
+    if (writable === false) throw SESSION_NOT_WRITABLE()
+    if (this._batch && !force) throw BATCH_UNFLUSHED()
+
+    await this.core.truncate(newLength, fork, { keyPair, signature })
 
     // TODO: Should propagate from an event triggered by the oplog
     this.replicator.updateAll()
   }
 
-  async append (blocks, opts) {
+  async append (blocks, opts = {}) {
     if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()
 
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw SESSION_NOT_WRITABLE()
+
+    const { keyPair = this.keyPair, signature = null } = opts
+    const writable = !this._readonly && !!(signature || (keyPair && keyPair.secretKey))
+
+    if (writable === false) throw SESSION_NOT_WRITABLE()
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
@@ -961,7 +987,7 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return this.core.append(buffers, (opts && opts.auth) || this.auth, { preappend })
+    return this.core.append(buffers, { keyPair, signature, preappend })
   }
 
   async treeHash (length) {

package/lib/batch.js

@@ -1,4 +1,4 @@
-const { SESSION_NOT_WRITABLE, BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
+const { BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
 const EventEmitter = require('events')
 const c = require('compact-encoding')
 
@@ -11,11 +11,12 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this.closed = false
     this.opening = null
     this.closing = null
+    this.writable = true // always writable...
     this.autoClose = autoClose
+    this.fork = 0
 
     this._appends = []
     this._byteLength = 0
-    this._fork = 0
     this._sessionLength = 0
     this._sessionByteLength = 0
     this._flushing = null
@@ -51,16 +52,12 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return this._sessionByteLength + this._byteLength
   }
 
-  get writable () {
-    return this.session.writable
-  }
-
   get core () {
     return this.session.core
   }
 
-  get auth () {
-    return this.session.auth
+  get manifest () {
+    return this.session.manifest
   }
 
   async ready () {
@@ -68,6 +65,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (this.opened) return
     this._sessionLength = this.session.length
     this._sessionByteLength = this.session.byteLength
+    this.fork = this.session.fork
     this.opened = true
     this.emit('ready')
   }
@@ -169,19 +167,30 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return b
   }
 
-  async truncate (newLength) {
+  async truncate (newLength = 0, opts = {}) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw SESSION_NOT_WRITABLE()
     if (this.closing) throw SESSION_CLOSED()
 
     // wait for any pending flush... (prop needs a lock)
     await this._waitForFlush()
 
+    if (typeof opts === 'number') opts = { fork: opts }
+    const { fork = this.fork + 1, force = false } = opts
+
     const length = this._sessionLength
-    if (newLength < length) throw new Error('Cannot truncate committed blocks')
+    if (newLength < length) {
+      if (!force) throw new Error('Cannot truncate committed blocks')
+      this._appends.length = 0
+      this._byteLength = 0
+      await this.session.truncate(newLength, { fork, force: true, ...opts })
+      this._sessionLength = this.session.length
+      this._sessionByteLength = this.session.byteLength
+    } else {
+      for (let i = newLength - length; i < this._appends.length; i++) this._byteLength -= this._appends[i].byteLength
+      this._appends.length = newLength - length
+    }
 
-    this._appends.length = newLength - length
-    this._fork++
+    this.fork = fork
 
     this.emit('truncate', newLength, this.fork)
   }
@@ -190,7 +199,6 @@ module.exports = class HypercoreBatch extends EventEmitter {
     const session = this.session
 
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw SESSION_NOT_WRITABLE()
     if (this.closing) throw SESSION_CLOSED()
 
     // wait for any pending flush... (prop needs a lock)
@@ -218,12 +226,14 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return info
   }
 
-  async flush (length = this._appends.length, auth) {
+  async flush (opts = {}) {
     if (this.opened === false) await this.opening
     if (this.closing) throw SESSION_CLOSED()
 
+    const { length = this._appends.length, keyPair = this.session.keyPair, signature = null } = opts
+
     while (this._flushing) await this._flushing
-    this._flushing = this._flush(length, auth)
+    this._flushing = this._flush(length, keyPair, signature)
 
     try {
       await this._flushing
@@ -234,11 +244,12 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (this.autoClose) await this.close()
   }
 
-  async _flush (length, auth) { // TODO: make this safe to interact with a parallel truncate...
+  async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
     if (this._appends.length === 0) return
 
     const flushingLength = Math.min(length, this._appends.length)
-    const info = await this.session.append(flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends, { auth })
+    const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
+    const info = await this.session.append(blocks, { keyPair, signature })
     const delta = info.byteLength - this._sessionByteLength
 
     this._sessionLength = info.length
@@ -267,7 +278,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
   _clearAppends () {
     this._appends = []
     this._byteLength = 0
-    this._fork = 0
+    this.fork = 0
   }
 
   _clearBatch () {

package/lib/caps.js

@@ -8,6 +8,7 @@ const c = require('compact-encoding')
 
 const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER, MANIFEST, DEFAULT_NAMESPACE] = crypto.namespace('hypercore', 5)
 
+exports.MANIFEST = MANIFEST
 exports.DEFAULT_NAMESPACE = DEFAULT_NAMESPACE
 
 exports.replicate = function (isInitiator, key, handshakeHash) {
@@ -16,27 +17,20 @@ exports.replicate = function (isInitiator, key, handshakeHash) {
   return out
 }
 
-exports.manifestHash = function (manifest) {
-  const state = { start: 0, end: 32 + manifest.byteLength, buffer: null }
-  state.buffer = b4a.allocUnsafe(state.end)
-  c.raw.encode(state, MANIFEST)
-  c.raw.encode(state, manifest)
-  const out = b4a.allocUnsafe(32)
-  sodium.crypto_generichash(out, state.buffer)
-  return out
-}
-
-exports.treeSignable = function (hash, length, fork) {
-  const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
+exports.treeSignable = function (namespace, hash, length, fork) {
+  const state = { start: 0, end: 112, buffer: b4a.allocUnsafe(112) }
   c.raw.encode(state, TREE)
+  c.raw.encode(state, namespace)
   c.raw.encode(state, hash)
   c.uint64.encode(state, length)
   c.uint64.encode(state, fork)
   return state.buffer
 }
 
-exports.treeSignableLegacy = function (hash, length, fork) {
-  const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
+exports.treeSignableCompat = function (hash, length, fork, noHeader) {
+  const end = noHeader ? 48 : 80
+  const state = { start: 0, end, buffer: b4a.allocUnsafe(end) }
+  if (!noHeader) c.raw.encode(state, TREE) // ultra legacy mode, kill in future major
   c.raw.encode(state, hash)
   c.uint64.encode(state, length)
   c.uint64.encode(state, fork)