hypercore 10.18.4 → 10.19.0

package/index.js

@@ -658,6 +658,10 @@ module.exports = class Hypercore extends EventEmitter {
     return null
   }
 
+  createTreeBatch () {
+    return this.core.tree.batch()
+  }
+
   findingPeers () {
     this._findingPeers++
     if (this.replicator !== null && !this.closing) this.replicator.findingPeers++
@@ -709,9 +713,9 @@ module.exports = class Hypercore extends EventEmitter {
     return true
   }
 
-  batch () {
+  batch ({ autoClose = true, session = true } = {}) {
     if (this._batch !== null) throw BATCH_ALREADY_EXISTS()
-    const batch = new Batch(this)
+    const batch = new Batch(session ? this.session() : this, autoClose)
     for (const session of this.sessions) session._batch = batch
     return batch
   }
@@ -878,7 +882,8 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   async truncate (newLength = 0, fork = -1) {
-    if (this._batch && !this._batch.flushed) throw BATCH_UNFLUSHED()
+    if (this._batch) throw BATCH_UNFLUSHED()
+
     if (this.opened === false) await this.opening
     if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
@@ -889,8 +894,9 @@ module.exports = class Hypercore extends EventEmitter {
     this.replicator.updateAll()
   }
 
-  async append (blocks) {
-    if (this._batch && !this._batch.flushed) throw BATCH_UNFLUSHED()
+  async append (blocks, opts) {
+    if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()
+
     if (this.opened === false) await this.opening
     if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
@@ -906,7 +912,7 @@ module.exports = class Hypercore extends EventEmitter {
       }
     }
 
-    return this.core.append(buffers, this.auth, { preappend })
+    return this.core.append(buffers, (opts && opts.auth) || this.auth, { preappend })
   }
 
   async treeHash (length) {

package/lib/batch.js

@@ -1,66 +1,196 @@
-const { SESSION_NOT_WRITABLE, BATCH_ALREADY_FLUSHED } = require('hypercore-errors')
+const { SESSION_NOT_WRITABLE, BLOCK_NOT_AVAILABLE, SESSION_CLOSED } = require('hypercore-errors')
+const EventEmitter = require('events')
+const c = require('compact-encoding')
+
+module.exports = class HypercoreBatch extends EventEmitter {
+  constructor (session, autoClose) {
+    super()
 
-module.exports = class Batch {
-  constructor (session) {
     this.session = session
-    this.flushed = false
+    this.opened = false
+    this.closed = false
+    this.opening = null
+    this.closing = null
+    this.autoClose = autoClose
 
     this._appends = []
     this._byteLength = 0
+    this._fork = 0
+    this._sessionLength = 0
+    this._sessionByteLength = 0
+    this._flushing = null
+
+    this.opening = this.ready().catch(noop)
+  }
+
+  get id () {
+    return this.session.id
+  }
+
+  get key () {
+    return this.session.key
+  }
+
+  get discoveryKey () {
+    return this.session.discoveryKey
+  }
+
+  get indexedLength () {
+    return this._sessionLength
+  }
+
+  get indexedByteLength () {
+    return this._sessionByteLength
   }
 
   get length () {
-    return this.session.length + this._appends.length
+    return this._sessionLength + this._appends.length
+  }
+
+  get byteLength () {
+    return this._sessionByteLength + this._byteLength
+  }
+
+  get writable () {
+    return this.session.writable
+  }
+
+  get core () {
+    return this.session.core
+  }
+
+  async ready () {
+    await this.session.ready()
+    if (this.opened) return
+    this._sessionLength = this.session.length
+    this._sessionByteLength = this.session.byteLength
+    this.opened = true
+    this.emit('ready')
+  }
+
+  async update (opts) {
+    if (this.opened === false) await this.ready()
+    await this.session.update(opts)
   }
 
-  ready () {
-    return this.session.ready()
+  setUserData (key, value, opts) {
+    return this.session.setUserData(key, value, opts)
+  }
+
+  getUserData (key, opts) {
+    return this.session.getUserData(key, opts)
   }
 
   async info (opts) {
     const session = this.session
     const info = await session.info(opts)
 
-    if (info.contiguousLength === info.length) {
+    info.length = this._sessionLength
+
+    if (info.contiguousLength >= info.length) {
       info.contiguousLength = info.length += this._appends.length
     } else {
       info.length += this._appends.length
     }
 
-    info.byteLength += this._byteLength
+    info.byteLength = this._sessionByteLength + this._byteLength
 
     return info
   }
 
+  async seek (bytes, opts) {
+    if (this.opened === false) await this.opening
+    if (this.closing) throw SESSION_CLOSED()
+
+    if (bytes < this._sessionByteLength) return await this.session.seek(bytes, opts)
+
+    bytes -= this._sessionByteLength
+
+    let i = 0
+
+    for (const blk of this._appends) {
+      if (bytes < blk.byteLength) return [this._sessionLength + i, bytes]
+      i++
+      bytes -= blk.byteLength
+    }
+
+    if (bytes === 0) return [this._sessionLength + i, 0]
+
+    throw BLOCK_NOT_AVAILABLE()
+  }
+
   async get (index, opts) {
-    const session = this.session
-    if (session.opened === false) await session.opening
+    if (this.opened === false) await this.opening
+    if (this.closing) throw SESSION_CLOSED()
 
-    const length = this.session.length
+    const length = this._sessionLength
     if (index < length) return this.session.get(index, opts)
 
-    return this._appends[index - length] || null
+    const buffer = this._appends[index - length] || null
+    if (!buffer) throw BLOCK_NOT_AVAILABLE()
+
+    const encoding = (opts && opts.valueEncoding && c.from(opts.valueEncoding)) || this.session.valueEncoding
+    if (!encoding) return buffer
+
+    return c.decode(encoding, buffer)
+  }
+
+  async _waitForFlush () {
+    // wait for any pending flush...
+    while (this._flushing) {
+      await this._flushing
+      await Promise.resolve() // yield in case a new flush is queued
+    }
+  }
+
+  createTreeBatch (length, blocks = []) {
+    if (!length && length !== 0) length = this.length + blocks.length
+
+    const maxLength = this.length + blocks.length
+    const b = this.session.createTreeBatch()
+    const len = Math.min(length, this.length)
+
+    if (len < this._sessionLength || length > maxLength) return null
+
+    for (let i = 0; i < len - this._sessionLength; i++) {
+      b.append(this._appends[i])
+    }
+
+    if (len < this.length) return b
+
+    for (let i = 0; i < length - len; i++) {
+      b.append(blocks[i])
+    }
+
+    return b
   }
 
   async truncate (newLength) {
-    if (this.flushed) throw BATCH_ALREADY_FLUSHED()
+    if (this.opened === false) await this.opening
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
+    if (this.closing) throw SESSION_CLOSED()
 
-    const session = this.session
-    if (session.opened === false) await session.opening
-    if (session.writable === false) throw SESSION_NOT_WRITABLE()
+    // wait for any pending flush... (prop needs a lock)
+    await this._waitForFlush()
 
-    const length = session.length
+    const length = this._sessionLength
     if (newLength < length) throw new Error('Cannot truncate committed blocks')
 
     this._appends.length = newLength - length
+    this._fork++
+
+    this.emit('truncate', newLength, this.fork)
   }
 
   async append (blocks) {
-    if (this.flushed) throw BATCH_ALREADY_FLUSHED()
-
     const session = this.session
-    if (session.opened === false) await session.opening
-    if (session.writable === false) throw SESSION_NOT_WRITABLE()
+
+    if (this.opened === false) await this.opening
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
+    if (this.closing) throw SESSION_CLOSED()
+
+    // wait for any pending flush... (prop needs a lock)
+    await this._waitForFlush()
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
@@ -78,37 +208,67 @@ module.exports = class Batch {
 
     this._appends.push(...buffers)
 
-    const byteLength = session.byteLength + this._byteLength
+    const info = { length: this.length, byteLength: this.byteLength }
+    this.emit('append')
 
-    return { length: this.length, byteLength }
+    return info
   }
 
-  async flush () {
-    if (this.flushed) throw BATCH_ALREADY_FLUSHED()
-    this.flushed = true
+  async flush (length = this._appends.length, auth) {
+    if (this.opened === false) await this.opening
+    if (this.closing) throw SESSION_CLOSED()
+
+    while (this._flushing) await this._flushing
+    this._flushing = this._flush(length, auth)
 
     try {
-      if (this._appends.length) await this.session.append(this._appends)
+      await this._flushing
     } finally {
-      this._clearBatch()
-      this._clearAppends()
+      this._flushing = null
     }
+
+    if (this.autoClose) await this.close()
+  }
+
+  async _flush (length, auth) { // TODO: make this safe to interact with a parallel truncate...
+    if (this._appends.length === 0) return
+
+    const flushingLength = Math.min(length, this._appends.length)
+    const info = await this.session.append(flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends, { auth })
+    const delta = info.byteLength - this._sessionByteLength
+
+    this._sessionLength = info.length
+    this._sessionByteLength = info.byteLength
+    this._appends = this._appends.slice(flushingLength)
+    this._byteLength -= delta
+
+    this.emit('flush')
   }
 
-  async close () {
-    if (this.flushed) throw BATCH_ALREADY_FLUSHED()
-    this.flushed = true
+  close () {
+    if (!this.closing) this.closing = this._close()
+    return this.closing
+  }
 
+  async _close () {
     this._clearBatch()
     this._clearAppends()
+
+    await this.session.close()
+
+    this.closed = true
+    this.emit('close')
   }
 
   _clearAppends () {
     this._appends = []
     this._byteLength = 0
+    this._fork = 0
   }
 
   _clearBatch () {
     for (const session of this.session.sessions) session._batch = null
   }
 }
+
+function noop () {}

package/lib/core.js

@@ -232,7 +232,7 @@ module.exports = class Core {
 
     try {
       const batch = await this.tree.truncate(length, fork)
-      batch.signature = await auth.sign(batch.signable())
+      batch.signature = await auth.sign(batch.signable(), batch)
       await this._truncate(batch, null)
     } finally {
       this.truncating--
@@ -322,7 +322,7 @@ module.exports = class Core {
       for (const val of values) batch.append(val)
 
       const hash = batch.hash()
-      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
+      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash), batch)
 
       const entry = {
         userData: null,
@@ -359,7 +359,7 @@ module.exports = class Core {
 
   _signed (batch, hash, auth = this.defaultAuth) {
     const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
-    return auth.verify(signable, batch.signature)
+    return auth.verify(signable, batch.signature, batch)
   }
 
   async _verifyExclusive ({ batch, bitfield, value, from }) {

package/lib/merkle-tree.js

@@ -85,6 +85,34 @@ class MerkleTreeBatch {
     return caps.treeSignableLegacy(hash, this.length, this.fork)
   }
 
+  get (index) {
+    if (index >= this.length * 2) {
+      return null
+    }
+
+    if (index < this.treeLength * 2) {
+      return this.tree.get(index)
+    }
+
+    for (const n of this.nodes) {
+      if (n.index === index) return n
+    }
+
+    return null
+  }
+
+  proof ({ block, hash, seek, upgrade }) {
+    return generateProof(this, block, hash, seek, upgrade)
+  }
+
+  verifyUpgrade (proof) {
+    const unverified = verifyTree(proof, this.tree.crypto, this.nodes)
+
+    if (!proof.upgrade) throw INVALID_OPERATION('Expected upgrade proof')
+
+    return verifyUpgrade(proof, unverified, this)
+  }
+
   append (buf) {
     const head = this.length * 2
     const ite = flat.iterator(head)
@@ -640,80 +668,8 @@ module.exports = class MerkleTree {
     return batch
   }
 
-  async proof ({ block, hash, seek, upgrade }) {
-    // Important that this does not throw inbetween making the promise arrays
-    // and finalise being called, otherwise there will be lingering promises in the background
-
-    const fork = this.fork
-    const signature = this.signature
-    const head = 2 * this.length
-    const from = upgrade ? upgrade.start * 2 : 0
-    const to = upgrade ? from + upgrade.length * 2 : head
-    const node = normalizeIndexed(block, hash)
-
-    if (from >= to || to > head) {
-      throw INVALID_OPERATION('Invalid upgrade')
-    }
-    if (seek && upgrade && node !== null && node.index >= from) {
-      throw INVALID_OPERATION('Cannot both do a seek and block/hash request when upgrading')
-    }
-
-    let subTree = head
-
-    const p = {
-      node: null,
-      seek: null,
-      upgrade: null,
-      additionalUpgrade: null
-    }
-
-    if (node !== null && (!upgrade || node.lastIndex < upgrade.start)) {
-      subTree = nodesToRoot(node.index, node.nodes, to)
-      const seekRoot = seek ? await seekUntrustedTree(this, subTree, seek.bytes) : head
-      blockAndSeekProof(this, node, seek, seekRoot, subTree, p)
-    } else if ((node || seek) && upgrade) {
-      subTree = seek ? await seekFromHead(this, to, seek.bytes) : node.index
-    }
-
-    if (upgrade) {
-      upgradeProof(this, node, seek, from, to, subTree, p)
-      if (head > to) additionalUpgradeProof(this, to, head, p)
-    }
-
-    const [pNode, pSeek, pUpgrade, pAdditional] = await settleProof(p)
-    const result = { fork, block: null, hash: null, seek: null, upgrade: null }
-
-    if (block) {
-      result.block = {
-        index: block.index,
-        value: null, // populated upstream, alloc it here for simplicity
-        nodes: pNode
-      }
-    } else if (hash) {
-      result.hash = {
-        index: hash.index,
-        nodes: pNode
-      }
-    }
-
-    if (seek && pSeek !== null) {
-      result.seek = {
-        bytes: seek.bytes,
-        nodes: pSeek
-      }
-    }
-
-    if (upgrade) {
-      result.upgrade = {
-        start: upgrade.start,
-        length: upgrade.length,
-        nodes: pUpgrade,
-        additionalNodes: pAdditional || [],
-        signature
-      }
-    }
-
-    return result
+  proof ({ block, hash, seek, upgrade }) {
+    return generateProof(this, block, hash, seek, upgrade)
   }
 
   // Successor to .nodes()
@@ -1216,3 +1172,79 @@ async function settleProof (p) {
     throw err
   }
 }
+
+// tree can be either the merkle tree or a merkle tree batch
+async function generateProof (tree, block, hash, seek, upgrade) {
+  // Important that this does not throw inbetween making the promise arrays
+  // and finalise being called, otherwise there will be lingering promises in the background
+
+  const fork = tree.fork
+  const signature = tree.signature
+  const head = 2 * tree.length
+  const from = upgrade ? upgrade.start * 2 : 0
+  const to = upgrade ? from + upgrade.length * 2 : head
+  const node = normalizeIndexed(block, hash)
+  if (from >= to || to > head) {
+    throw INVALID_OPERATION('Invalid upgrade')
+  }
+  if (seek && upgrade && node !== null && node.index >= from) {
+    throw INVALID_OPERATION('Cannot both do a seek and block/hash request when upgrading')
+  }
+
+  let subTree = head
+
+  const p = {
+    node: null,
+    seek: null,
+    upgrade: null,
+    additionalUpgrade: null
+  }
+
+  if (node !== null && (!upgrade || node.lastIndex < upgrade.start)) {
+    subTree = nodesToRoot(node.index, node.nodes, to)
+    const seekRoot = seek ? await seekUntrustedTree(tree, subTree, seek.bytes) : head
+    blockAndSeekProof(tree, node, seek, seekRoot, subTree, p)
+  } else if ((node || seek) && upgrade) {
+    subTree = seek ? await seekFromHead(tree, to, seek.bytes) : node.index
+  }
+
+  if (upgrade) {
+    upgradeProof(tree, node, seek, from, to, subTree, p)
+    if (head > to) additionalUpgradeProof(tree, to, head, p)
+  }
+
+  const [pNode, pSeek, pUpgrade, pAdditional] = await settleProof(p)
+  const result = { fork, block: null, hash: null, seek: null, upgrade: null }
+
+  if (block) {
+    result.block = {
+      index: block.index,
+      value: null, // populated upstream, alloc it here for simplicity
+      nodes: pNode
+    }
+  } else if (hash) {
+    result.hash = {
+      index: hash.index,
+      nodes: pNode
+    }
+  }
+
+  if (seek && pSeek !== null) {
+    result.seek = {
+      bytes: seek.bytes,
+      nodes: pSeek
+    }
+  }
+
+  if (upgrade) {
+    result.upgrade = {
+      start: upgrade.start,
+      length: upgrade.length,
+      nodes: pUpgrade,
+      additionalNodes: pAdditional || [],
+      signature
+    }
+  }
+
+  return result
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.18.4",
+  "version": "10.19.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {