hypercore 10.36.3 → 10.37.0

package/index.js

@@ -18,7 +18,7 @@ const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const Batch = require('./lib/batch')
-const { manifestHash, defaultSignerManifest, createManifest, isCompat, sign } = require('./lib/verifier')
+const { manifestHash, createManifest } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const {
   ASSERTION,
@@ -85,7 +85,6 @@ module.exports = class Hypercore extends EventEmitter {
     this.closing = null
     this.opening = null
 
-    this._clone = opts.clone || null
     this._readonly = opts.writable === false
     this._preappend = preappend.bind(this)
     this._snapshot = null
@@ -322,14 +321,6 @@ module.exports = class Hypercore extends EventEmitter {
           const s = this.sessions[i]
           if (s !== this) s._passCapabilities(this)
         }
-
-        // copy state over
-        if (this._clone) {
-          const { from, signature } = this._clone
-          await from.opening
-          await this.core.copyFrom(from.core, signature)
-          this._clone = null
-        }
       }
     } else {
       ensureEncryption(this, opts)
@@ -501,46 +492,6 @@ module.exports = class Hypercore extends EventEmitter {
     this.emit('close', true)
   }
 
-  clone (keyPair, storage, opts = {}) {
-    // TODO: current limitation is no forking
-    if ((opts.fork && opts.fork !== 0) || this.fork !== 0) {
-      throw BAD_ARGUMENT('Cannot clone a fork')
-    }
-
-    const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
-    const key = opts.key || (opts.compat !== false ? manifest.signers[0].publicKey : manifestHash(manifest))
-
-    if (b4a.equals(key, this.key)) {
-      throw BAD_ARGUMENT('Clone cannot share verification information')
-    }
-
-    const signature = opts.signature === undefined
-      ? sign(createManifest(manifest), this.core.tree.batch(), keyPair, { compat: isCompat(key, manifest) })
-      : opts.signature
-
-    const sparse = opts.sparse === false ? false : this.sparse
-    const wait = opts.wait === false ? false : this.wait
-    const onwait = opts.onwait === undefined ? this.onwait : opts.onwait
-    const timeout = opts.timeout === undefined ? this.timeout : opts.timeout
-
-    const Clz = this.constructor
-
-    return new Clz(storage, key, {
-      ...opts,
-      keyPair,
-      sparse,
-      wait,
-      onwait,
-      timeout,
-      manifest,
-      overwrite: true,
-      clone: {
-        from: this,
-        signature
-      }
-    })
-  }
-
   replicate (isInitiator, opts = {}) {
     // Only limitation here is that ondiscoverykey doesn't work atm when passing a muxer directly,
     // because it doesn't really make a lot of sense.

package/lib/core.js

@@ -253,7 +253,7 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature, { length = src.tree.length, sourceLength = src.tree.length, fork = src.tree.fork, additional = [] } = {}) {
+  async copyPrologue (src, { additional = [] } = {}) {
     await this._mutex.lock()
 
     try {
@@ -263,120 +263,103 @@ module.exports = class Core {
       throw err
     }
 
-    const initialLength = this.tree.length
-
     try {
-      const updates = []
-
-      let pos = 0
-      const copyLength = Math.min(src.tree.length, length)
-
-      while (pos < copyLength) {
-        const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
-        if (segmentStart >= length || segmentStart < 0) break
+      const prologue = this.header.manifest && this.header.manifest.prologue
+      if (!prologue) throw INVALID_OPERATION('No prologue present')
 
-        const segmentEnd = Math.min(src.tree.length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
+      const srcLength = prologue.length - additional.length
+      const srcBatch = srcLength !== src.tree.length ? await src.tree.truncate(srcLength) : src.tree.batch()
+      const srcRoots = srcBatch.roots.slice(0)
+      const srcByteLength = srcBatch.byteLength
 
-        const segment = []
+      for (const blk of additional) srcBatch.append(blk)
 
-        pos = segmentStart
-        while (pos < segmentEnd) {
-          const val = await src.blocks.get(pos++)
-          segment.push(val)
-        }
+      if (!b4a.equals(srcBatch.hash(), prologue.hash)) throw INVALID_OPERATION('Source tree is conflicting')
 
-        const [offset] = await src.tree.byteRange(2 * segmentStart)
-        await this.blocks.putBatch(segmentStart, segment, offset)
+      // all hashes are correct, lets copy
 
-        this._setBitfieldRange(segmentStart, segmentEnd - segmentStart, true)
-
-        pos = segmentEnd + 1
-
-        updates.push({
-          drop: false,
-          start: segmentStart,
-          length: segmentEnd - segmentStart
-        })
+      const entry = {
+        userData: null,
+        treeNodes: srcRoots,
+        treeUpgrade: null,
+        bitfield: null
       }
 
-      for (let i = 0; i < copyLength * 2; i++) {
-        const node = await src.tree.get(i, false)
-        if (node === null) continue
-
-        await this.tree.addNode(node)
+      if (additional.length) {
+        await this.blocks.putBatch(srcLength, additional, srcByteLength)
+        entry.treeNodes = entry.treeNodes.concat(srcBatch.nodes)
+        entry.bitfield = {
+          drop: false,
+          start: srcLength,
+          length: additional.length
+        }
       }
 
-      await this.tree.flush()
+      await this.oplog.append([entry], false)
+      this.tree.addNodes(entry.treeNodes)
 
-      this.tree.fork = fork
+      if (this.header.tree.length < srcBatch.length) {
+        this.header.tree.length = srcBatch.length
+        this.header.tree.rootHash = srcBatch.hash()
 
-      let batch = this.tree.batch()
+        this.tree.length = srcBatch.length
+        this.tree.byteLength = srcBatch.byteLength
+        this.tree.roots = srcBatch.roots
+        this.onupdate(0b0001, null, null, null)
+      }
 
-      // add additional blocks
-      if (length > src.tree.length) {
-        const missing = length - src.tree.length
-        const offset = src.tree.length - sourceLength
+      if (entry.bitfield) {
+        this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)
+        this.onupdate(0, entry.bitfield, null, null)
+      }
 
-        if (additional.length < missing + offset) {
-          throw INVALID_OPERATION('Insufficient additional nodes were passed')
-        }
+      await this._flushOplog()
 
-        const source = src.tree.batch()
+      // no more additional blocks now and we should be consistant on disk
+      // copy over all existing segments...
 
-        batch.roots = [...source.roots]
-        batch.length = source.length
-        batch.byteLength = source.byteLength
+      let segmentEnd = 0
 
-        const blocks = additional.length === missing ? additional : additional.slice(offset, offset + missing)
+      while (segmentEnd < srcLength) {
+        const segmentStart = maximumSegmentStart(segmentEnd, src.bitfield, this.bitfield)
+        if (segmentStart >= srcLength || segmentStart < 0) break
 
-        await this.blocks.putBatch(source.length, blocks, source.byteLength)
-        this._setBitfieldRange(source.length, missing, true)
+        // max segment is 65536 to avoid running out of memory
+        segmentEnd = Math.min(segmentStart + 65536, srcLength, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
 
-        updates.push({
+        const treeNodes = await src.tree.getNeededNodes(srcLength, segmentStart, segmentEnd)
+        const bitfield = {
           drop: false,
-          start: source.length,
-          length: missing
-        })
-
-        for (const block of blocks) await batch.append(block)
-      } else {
-        const source = length < src.tree.length ? await src.tree.truncate(length) : src.tree.batch()
+          start: segmentStart,
+          length: segmentEnd - segmentStart
+        }
 
-        this.tree.roots = [...source.roots]
-        this.tree.length = source.length
-        this.tree.byteLength = source.byteLength
+        const segment = []
+        for (let i = segmentStart; i < segmentEnd; i++) {
+          const blk = await src.blocks.get(i)
+          segment.push(blk)
+        }
 
-        // update batch
-        batch = this.tree.batch()
-      }
+        const offset = await src.tree.byteOffset(2 * segmentStart)
+        await this.blocks.putBatch(segmentStart, segment, offset)
 
-      // verify if upgraded
-      if (batch.length > initialLength) {
-        try {
-          batch.signature = signature
-
-          this._verifyBatchUpgrade(batch, this.header.manifest)
-          this.tree.signature = signature
-        } catch (err) {
-          this.tree.signature = null
-          // TODO: how to handle signature failure?
-          throw err
+        const entry = {
+          userData: null,
+          treeNodes,
+          treeUpgrade: null,
+          bitfield
         }
-      }
 
-      await batch.commit()
-
-      this.header.tree.length = this.tree.length
-      this.header.tree.fork = this.tree.fork
-      this.header.tree.rootHash = this.tree.hash()
-      this.header.tree.signature = this.tree.signature
+        await this.oplog.append([entry], false)
+        this.tree.addNodes(treeNodes)
+        this._setBitfieldRange(bitfield.start, bitfield.length, true)
+        this.onupdate(0, bitfield, null, null)
+        await this._flushOplog()
+      }
 
       this.header.userData = src.header.userData.slice(0)
-      this.header.hints.contiguousLength = Math.min(src.header.hints.contiguousLength, this.header.tree.length)
-
-      for (const bitfield of updates) {
-        this.onupdate(0b0001, bitfield, null, null)
-      }
+      const contig = Math.min(src.header.hints.contiguousLength, srcBatch.length)
+      if (this.header.hints.contiguousLength < contig) this.header.hints.contiguousLength = contig
 
       await this._flushOplog()
     } finally {
@@ -615,7 +598,7 @@ module.exports = class Core {
         }
       }
 
-      await this.blocks.putBatch(treeLength, adding > values.length ? values.slice(0, adding) : values, byteOffset)
+      await this.blocks.putBatch(treeLength, adding < values.length ? values.slice(0, adding) : values, byteOffset)
       await this.oplog.append([entry], false)
 
       this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)

package/lib/merkle-tree.js

@@ -4,7 +4,7 @@ const c = require('compact-encoding')
 const Xache = require('xache')
 const b4a = require('b4a')
 const caps = require('./caps')
-const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT } = require('hypercore-errors')
+const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT, ASSERTION } = require('hypercore-errors')
 
 const BLANK_HASH = b4a.alloc(32)
 const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
@@ -429,6 +429,37 @@ module.exports = class MerkleTree {
     this.prologue = { hash, length }
   }
 
+  addNodes (nodes) {
+    for (let i = 0; i < nodes.length; i++) {
+      const node = nodes[i]
+      this.unflushed.set(node.index, node)
+    }
+  }
+
+  getNeededNodes (length, start, end) {
+    const nodes = new Map()
+    const head = length * 2
+
+    for (let i = start; i < end; i++) {
+      const ite = flat.iterator(i * 2)
+
+      while (true) {
+        if (nodes.has(ite.index)) break
+        nodes.set(ite.index, this.get(ite.index, true))
+
+        const sibling = ite.sibling()
+
+        ite.parent()
+        if (ite.contains(head)) break
+
+        if (nodes.has(sibling)) break
+        nodes.set(sibling, this.get(sibling, true))
+      }
+    }
+
+    return Promise.all([...nodes.values()])
+  }
+
   async upgradeable (length) {
     const indexes = flat.fullRoots(2 * length)
     const roots = new Array(indexes.length)
@@ -767,6 +798,8 @@ async function getByteOffset (tree, index) {
 
     return offset
   }
+
+  throw ASSERTION('Failed to find offset')
 }
 
 // All the methods needed for proof verification
@@ -824,7 +857,7 @@ function verifyUpgrade ({ fork, upgrade }, blockRoot, batch) {
 
   if (prologue) {
     const { start, length } = upgrade
-    if (start < prologue.length && (start !== 0 || length !== prologue.length)) {
+    if (start < prologue.length && (start !== 0 || length < prologue.length)) {
       throw INVALID_PROOF('Upgrade does not satisfy prologue')
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.36.3",
+  "version": "10.37.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {