hypercore 10.36.4 → 10.37.1

package/index.js

@@ -18,7 +18,7 @@ const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const Batch = require('./lib/batch')
-const { manifestHash, defaultSignerManifest, createManifest, isCompat, sign } = require('./lib/verifier')
+const { manifestHash, createManifest } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const {
   ASSERTION,
@@ -85,7 +85,6 @@ module.exports = class Hypercore extends EventEmitter {
     this.closing = null
     this.opening = null
 
-    this._clone = opts.clone || null
     this._readonly = opts.writable === false
     this._preappend = preappend.bind(this)
     this._snapshot = null
@@ -322,14 +321,6 @@ module.exports = class Hypercore extends EventEmitter {
           const s = this.sessions[i]
           if (s !== this) s._passCapabilities(this)
         }
-
-        // copy state over
-        if (this._clone) {
-          const { from, signature } = this._clone
-          await from.opening
-          await this.core.copyFrom(from.core, signature)
-          this._clone = null
-        }
       }
     } else {
       ensureEncryption(this, opts)
@@ -501,46 +492,6 @@ module.exports = class Hypercore extends EventEmitter {
     this.emit('close', true)
   }
 
-  clone (keyPair, storage, opts = {}) {
-    // TODO: current limitation is no forking
-    if ((opts.fork && opts.fork !== 0) || this.fork !== 0) {
-      throw BAD_ARGUMENT('Cannot clone a fork')
-    }
-
-    const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
-    const key = opts.key || (opts.compat !== false ? manifest.signers[0].publicKey : manifestHash(manifest))
-
-    if (b4a.equals(key, this.key)) {
-      throw BAD_ARGUMENT('Clone cannot share verification information')
-    }
-
-    const signature = opts.signature === undefined
-      ? sign(createManifest(manifest), this.core.tree.batch(), keyPair, { compat: isCompat(key, manifest) })
-      : opts.signature
-
-    const sparse = opts.sparse === false ? false : this.sparse
-    const wait = opts.wait === false ? false : this.wait
-    const onwait = opts.onwait === undefined ? this.onwait : opts.onwait
-    const timeout = opts.timeout === undefined ? this.timeout : opts.timeout
-
-    const Clz = this.constructor
-
-    return new Clz(storage, key, {
-      ...opts,
-      keyPair,
-      sparse,
-      wait,
-      onwait,
-      timeout,
-      manifest,
-      overwrite: true,
-      clone: {
-        from: this,
-        signature
-      }
-    })
-  }
-
   replicate (isInitiator, opts = {}) {
     // Only limitation here is that ondiscoverykey doesn't work atm when passing a muxer directly,
     // because it doesn't really make a lot of sense.

package/lib/batch.js

@@ -25,6 +25,7 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this._sessionLength = 0
     this._sessionByteLength = 0
     this._sessionBatch = null
+    this._cachedBatch = null
     this._flushing = null
     this._clear = clear
 
@@ -207,20 +208,32 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return this.session.core.tree.restoreBatch(length)
   }
 
-  createTreeBatch (length, blocks = []) {
+  _catchupBatch (clone) {
+    if (this._cachedBatch === null) this._cachedBatch = this._sessionBatch.clone()
+
+    if (this.length > this._cachedBatch.length) {
+      const offset = this._cachedBatch.length - this._sessionBatch.length
+
+      for (let i = offset; i < this._appendsActual.length; i++) {
+        this._cachedBatch.append(this._appendsActual[i])
+      }
+    }
+
+    return clone ? this._cachedBatch.clone() : this._cachedBatch
+  }
+
+  createTreeBatch (length, opts = {}) {
+    if (Array.isArray(opts)) opts = { blocks: opts }
+
+    const { blocks = [], clone = true } = opts
     if (!length && length !== 0) length = this.length + blocks.length
 
     const maxLength = this.length + blocks.length
-    const b = this._sessionBatch.clone()
+    const b = this._catchupBatch(clone || (blocks.length > 0 || length !== this.length))
     const len = Math.min(length, this.length)
 
     if (len < this._sessionLength || length > maxLength) return null
-
-    for (let i = 0; i < len - this._sessionLength; i++) {
-      b.append(this._appendsActual[i])
-    }
-
-    if (len < this.length) return b
+    if (len < b.length) b.checkout(len, this._sessionBatch.roots)
 
     for (let i = 0; i < length - len; i++) {
       b.append(this._appendsActual === this._appends ? blocks[i] : this._encrypt(b.length, blocks[i]))
@@ -239,6 +252,8 @@ module.exports = class HypercoreBatch extends EventEmitter {
     if (typeof opts === 'number') opts = { fork: opts }
     const { fork = this.fork + 1, force = false } = opts
 
+    this._cachedBatch = null
+
     const length = this._sessionLength
     if (newLength < length) {
       if (!force) throw new Error('Cannot truncate committed blocks')
@@ -380,6 +395,8 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this._sessionByteLength = info.byteLength
     this._sessionBatch = newBatch
 
+    if (this._cachedBatch !== null) this._cachedBatch.prune(info.length)
+
     const same = this._appends === this._appendsActual
 
     this._appends = this._appends.slice(flushingLength)

package/lib/core.js

@@ -253,7 +253,7 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature, { length = src.tree.length, sourceLength = src.tree.length, fork = src.tree.fork, additional = [] } = {}) {
+  async copyPrologue (src, { additional = [] } = {}) {
     await this._mutex.lock()
 
     try {
@@ -263,120 +263,103 @@ module.exports = class Core {
       throw err
     }
 
-    const initialLength = this.tree.length
-
     try {
-      const updates = []
-
-      let pos = 0
-      const copyLength = Math.min(src.tree.length, length)
-
-      while (pos < copyLength) {
-        const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
-        if (segmentStart >= length || segmentStart < 0) break
+      const prologue = this.header.manifest && this.header.manifest.prologue
+      if (!prologue) throw INVALID_OPERATION('No prologue present')
 
-        const segmentEnd = Math.min(src.tree.length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
+      const srcLength = prologue.length - additional.length
+      const srcBatch = srcLength !== src.tree.length ? await src.tree.truncate(srcLength) : src.tree.batch()
+      const srcRoots = srcBatch.roots.slice(0)
+      const srcByteLength = srcBatch.byteLength
 
-        const segment = []
+      for (const blk of additional) srcBatch.append(blk)
 
-        pos = segmentStart
-        while (pos < segmentEnd) {
-          const val = await src.blocks.get(pos++)
-          segment.push(val)
-        }
+      if (!b4a.equals(srcBatch.hash(), prologue.hash)) throw INVALID_OPERATION('Source tree is conflicting')
 
-        const [offset] = await src.tree.byteRange(2 * segmentStart)
-        await this.blocks.putBatch(segmentStart, segment, offset)
+      // all hashes are correct, lets copy
 
-        this._setBitfieldRange(segmentStart, segmentEnd - segmentStart, true)
-
-        pos = segmentEnd + 1
-
-        updates.push({
-          drop: false,
-          start: segmentStart,
-          length: segmentEnd - segmentStart
-        })
+      const entry = {
+        userData: null,
+        treeNodes: srcRoots,
+        treeUpgrade: null,
+        bitfield: null
       }
 
-      for (let i = 0; i < copyLength * 2; i++) {
-        const node = await src.tree.get(i, false)
-        if (node === null) continue
-
-        await this.tree.addNode(node)
+      if (additional.length) {
+        await this.blocks.putBatch(srcLength, additional, srcByteLength)
+        entry.treeNodes = entry.treeNodes.concat(srcBatch.nodes)
+        entry.bitfield = {
+          drop: false,
+          start: srcLength,
+          length: additional.length
+        }
       }
 
-      await this.tree.flush()
+      await this.oplog.append([entry], false)
+      this.tree.addNodes(entry.treeNodes)
 
-      this.tree.fork = fork
+      if (this.header.tree.length < srcBatch.length) {
+        this.header.tree.length = srcBatch.length
+        this.header.tree.rootHash = srcBatch.hash()
 
-      let batch = this.tree.batch()
+        this.tree.length = srcBatch.length
+        this.tree.byteLength = srcBatch.byteLength
+        this.tree.roots = srcBatch.roots
+        this.onupdate(0b0001, null, null, null)
+      }
 
-      // add additional blocks
-      if (length > src.tree.length) {
-        const missing = length - src.tree.length
-        const offset = src.tree.length - sourceLength
+      if (entry.bitfield) {
+        this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)
+        this.onupdate(0, entry.bitfield, null, null)
+      }
 
-        if (additional.length < missing + offset) {
-          throw INVALID_OPERATION('Insufficient additional nodes were passed')
-        }
+      await this._flushOplog()
 
-        const source = src.tree.batch()
+      // no more additional blocks now and we should be consistant on disk
+      // copy over all existing segments...
 
-        batch.roots = [...source.roots]
-        batch.length = source.length
-        batch.byteLength = source.byteLength
+      let segmentEnd = 0
 
-        const blocks = additional.length === missing ? additional : additional.slice(offset, offset + missing)
+      while (segmentEnd < srcLength) {
+        const segmentStart = maximumSegmentStart(segmentEnd, src.bitfield, this.bitfield)
+        if (segmentStart >= srcLength || segmentStart < 0) break
 
-        await this.blocks.putBatch(source.length, blocks, source.byteLength)
-        this._setBitfieldRange(source.length, missing, true)
+        // max segment is 65536 to avoid running out of memory
+        segmentEnd = Math.min(segmentStart + 65536, srcLength, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
 
-        updates.push({
+        const treeNodes = await src.tree.getNeededNodes(srcLength, segmentStart, segmentEnd)
+        const bitfield = {
           drop: false,
-          start: source.length,
-          length: missing
-        })
-
-        for (const block of blocks) await batch.append(block)
-      } else {
-        const source = length < src.tree.length ? await src.tree.truncate(length) : src.tree.batch()
+          start: segmentStart,
+          length: segmentEnd - segmentStart
+        }
 
-        this.tree.roots = [...source.roots]
-        this.tree.length = source.length
-        this.tree.byteLength = source.byteLength
+        const segment = []
+        for (let i = segmentStart; i < segmentEnd; i++) {
+          const blk = await src.blocks.get(i)
+          segment.push(blk)
+        }
 
-        // update batch
-        batch = this.tree.batch()
-      }
+        const offset = await src.tree.byteOffset(2 * segmentStart)
+        await this.blocks.putBatch(segmentStart, segment, offset)
 
-      // verify if upgraded
-      if (batch.length > initialLength) {
-        try {
-          batch.signature = signature
-
-          this._verifyBatchUpgrade(batch, this.header.manifest)
-          this.tree.signature = signature
-        } catch (err) {
-          this.tree.signature = null
-          // TODO: how to handle signature failure?
-          throw err
+        const entry = {
+          userData: null,
+          treeNodes,
+          treeUpgrade: null,
+          bitfield
         }
-      }
 
-      await batch.commit()
-
-      this.header.tree.length = this.tree.length
-      this.header.tree.fork = this.tree.fork
-      this.header.tree.rootHash = this.tree.hash()
-      this.header.tree.signature = this.tree.signature
+        await this.oplog.append([entry], false)
+        this.tree.addNodes(treeNodes)
+        this._setBitfieldRange(bitfield.start, bitfield.length, true)
+        this.onupdate(0, bitfield, null, null)
+        await this._flushOplog()
+      }
 
       this.header.userData = src.header.userData.slice(0)
-      this.header.hints.contiguousLength = Math.min(src.header.hints.contiguousLength, this.header.tree.length)
-
-      for (const bitfield of updates) {
-        this.onupdate(0b0001, bitfield, null, null)
-      }
+      const contig = Math.min(src.header.hints.contiguousLength, srcBatch.length)
+      if (this.header.hints.contiguousLength < contig) this.header.hints.contiguousLength = contig
 
       await this._flushOplog()
     } finally {

package/lib/merkle-tree.js

@@ -57,6 +57,68 @@ class MerkleTreeBatch {
     this.upgraded = false
   }
 
+  checkout (length, additionalRoots) {
+    const roots = []
+    let r = 0
+
+    const head = 2 * length - 2
+    const gaps = new Set()
+    const all = new Map()
+
+    // additional roots is so the original roots can be passed (we mutate the array in appendRoot)
+    if (additionalRoots) {
+      for (const node of additionalRoots) all.set(node.index, node)
+    }
+
+    for (const node of this.nodes) all.set(node.index, node)
+
+    for (const index of flat.fullRoots(head + 2)) {
+      const left = flat.leftSpan(index)
+      if (left !== 0) gaps.add(left - 1)
+
+      if (r < this.roots.length && this.roots[r].index === index) {
+        roots.push(this.roots[r++])
+        continue
+      }
+      const node = all.get(index)
+      if (!node) throw new BAD_ARGUMENT('root missing for given length')
+      roots.push(node)
+    }
+
+    this.roots = roots
+    this.length = length
+    this.byteLength = totalSize(roots)
+    this.hashCached = null
+    this.signature = null
+
+    for (let i = 0; i < this.nodes.length; i++) {
+      const index = this.nodes[i].index
+      if (index <= head && !gaps.has(index)) continue
+      const last = this.nodes.pop()
+      if (i < this.nodes.length) this.nodes[i--] = last
+    }
+  }
+
+  prune (length) {
+    if (length === 0) return
+
+    const head = 2 * length - 2
+    const gaps = new Set()
+
+    // TODO: make a function for this in flat-tree
+    for (const index of flat.fullRoots(head + 2)) {
+      const left = flat.leftSpan(index)
+      if (left !== 0) gaps.add(left - 1)
+    }
+
+    for (let i = 0; i < this.nodes.length; i++) {
+      const index = this.nodes[i].index
+      if (index > head || gaps.has(index)) continue
+      const last = this.nodes.pop()
+      if (i < this.nodes.length) this.nodes[i--] = last
+    }
+  }
+
   clone () {
     const b = new MerkleTreeBatch(this.tree)
 
@@ -429,6 +491,37 @@ module.exports = class MerkleTree {
     this.prologue = { hash, length }
   }
 
+  addNodes (nodes) {
+    for (let i = 0; i < nodes.length; i++) {
+      const node = nodes[i]
+      this.unflushed.set(node.index, node)
+    }
+  }
+
+  getNeededNodes (length, start, end) {
+    const nodes = new Map()
+    const head = length * 2
+
+    for (let i = start; i < end; i++) {
+      const ite = flat.iterator(i * 2)
+
+      while (true) {
+        if (nodes.has(ite.index)) break
+        nodes.set(ite.index, this.get(ite.index, true))
+
+        const sibling = ite.sibling()
+
+        ite.parent()
+        if (ite.contains(head)) break
+
+        if (nodes.has(sibling)) break
+        nodes.set(sibling, this.get(sibling, true))
+      }
+    }
+
+    return Promise.all([...nodes.values()])
+  }
+
   async upgradeable (length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
@@ -826,7 +919,7 @@ function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
 
   if (prologue) {
     const { start, length } = upgrade
-    if (start < prologue.length && (start !== 0 || length !== prologue.length)) {
+    if (start < prologue.length && (start !== 0 || length < prologue.length)) {
       throw INVALID_PROOF('Upgrade does not satisfy prologue')
     }
   }

package/lib/remote-bitfield.js

@@ -230,7 +230,11 @@ module.exports = class RemoteBitfield {
       i++
     }
 
-    return val ? -1 : this._maxSegments * BITS_PER_SEGMENT
+    // For the val === false case, we always return at least
+    // the 'position', also if nothing was found
+    return val
+      ? -1
+      : Math.max(position, this._maxSegments * BITS_PER_SEGMENT)
   }
 
   firstSet (position) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.36.4",
+  "version": "10.37.1",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {