hypercore 10.0.0-alpha.9 → 10.2.0

package/lib/core.js

@@ -1,13 +1,15 @@
 const hypercoreCrypto = require('hypercore-crypto')
+const b4a = require('b4a')
 const Oplog = require('./oplog')
 const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
-const { oplogHeader, oplogEntry } = require('./messages')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('./errors')
+const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, sign, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
     this.onupdate = onupdate
     this.header = header
     this.crypto = crypto
@@ -15,7 +17,7 @@ module.exports = class Core {
     this.tree = tree
     this.blocks = blocks
     this.bitfield = bitfield
-    this.defaultSign = sign
+    this.defaultAuth = auth
     this.truncating = 0
 
     this._maxOplogSize = 65536
@@ -23,6 +25,7 @@ module.exports = class Core {
     this._verifies = null
     this._verifiesFlushed = null
     this._mutex = new Mutex()
+    this._legacy = legacy
   }
 
   static async open (storage, opts = {}) {
@@ -34,42 +37,51 @@ module.exports = class Core {
     try {
       return await this.resume(oplogFile, treeFile, bitfieldFile, dataFile, opts)
     } catch (err) {
-      return new Promise((resolve, reject) => {
-        let missing = 4
-
-        oplogFile.close(done)
-        treeFile.close(done)
-        bitfieldFile.close(done)
-        dataFile.close(done)
-
-        function done () {
-          if (--missing === 0) reject(err)
-        }
-      })
+      await closeAll(oplogFile, treeFile, bitfieldFile, dataFile)
+      throw err
     }
   }
 
-  // TODO: we should prob have a general "auth" abstraction instead somewhere?
-  static createSigner (crypto, { publicKey, secretKey }) {
-    if (!crypto.validateKeyPair({ publicKey, secretKey })) throw new Error('Invalid key pair')
-    return signable => crypto.sign(signable, secretKey)
+  static createAuth (crypto, { publicKey, secretKey }, opts = {}) {
+    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) {
+      throw BAD_ARGUMENT('Invalid key pair')
+    }
+
+    const sign = opts.sign
+      ? opts.sign
+      : secretKey
+        ? (signable) => crypto.sign(signable, secretKey)
+        : undefined
+
+    return {
+      sign,
+      verify (signable, signature) {
+        return crypto.verify(signable, signature, publicKey)
+      }
+    }
   }
 
   static async resume (oplogFile, treeFile, bitfieldFile, dataFile, opts) {
-    const overwrite = opts.overwrite === true
+    let overwrite = opts.overwrite === true
+
+    const force = opts.force === true
     const createIfMissing = opts.createIfMissing !== false
     const crypto = opts.crypto || hypercoreCrypto
 
     const oplog = new Oplog(oplogFile, {
-      headerEncoding: oplogHeader,
-      entryEncoding: oplogEntry
+      headerEncoding: m.oplog.header,
+      entryEncoding: m.oplog.entry
     })
 
     let { header, entries } = await oplog.open()
 
-    if (!header || overwrite === true) {
+    if (force && opts.keyPair && header && header.signer && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      overwrite = true
+    }
+
+    if (!header || overwrite) {
       if (!createIfMissing) {
-        throw new Error('No hypercore is stored here')
+        throw STORAGE_EMPTY('No Hypercore is stored here')
       }
 
       header = {
@@ -84,14 +96,15 @@ module.exports = class Core {
         signer: opts.keyPair || crypto.keyPair(),
         hints: {
           reorgs: []
-        }
+        },
+        contiguousLength: 0
       }
 
       await oplog.flush(header)
     }
 
-    if (opts.keyPair && !header.signer.publicKey.equals(opts.keyPair.publicKey)) {
-      throw new Error('Another hypercore is stored here')
+    if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
+      throw STORAGE_CONFLICT('Another Hypercore is stored here')
     }
 
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
@@ -102,9 +115,18 @@ module.exports = class Core {
       await tree.clear()
       await blocks.clear()
       await bitfield.clear()
+      entries = []
+    } else if (bitfield.resumed && header.tree.length === 0) {
+      // If this was an old bitfield, reset it since it loads based on disk size atm (TODO: change that)
+      await bitfield.clear()
     }
 
-    const sign = opts.sign || (header.signer.secretKey ? this.createSigner(crypto, header.signer) : null)
+    // compat from earlier version that do not store contig length
+    if (header.contiguousLength === 0) {
+      while (bitfield.get(header.contiguousLength)) header.contiguousLength++
+    }
+
+    const auth = opts.auth || this.createAuth(crypto, header.signer)
 
     for (const e of entries) {
       if (e.userData) {
@@ -118,7 +140,8 @@ module.exports = class Core {
       }
 
       if (e.bitfield) {
-        bitfield.setRange(e.bitfield.start, e.bitfield.length)
+        bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
+        updateContig(header, e.bitfield, bitfield)
       }
 
       if (e.treeUpgrade) {
@@ -135,7 +158,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, sign, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -176,7 +199,7 @@ module.exports = class Core {
 
       for (const u of this.header.userData) {
         if (u.key !== key) continue
-        if (value && u.value.equals(value)) return
+        if (value && b4a.equals(u.value, value)) return
         empty = false
         break
       }
@@ -200,13 +223,13 @@ module.exports = class Core {
     }
   }
 
-  async truncate (length, fork, sign = this.defaultSign) {
+  async truncate (length, fork, auth = this.defaultAuth) {
     this.truncating++
     await this._mutex.lock()
 
     try {
       const batch = await this.tree.truncate(length, fork)
-      batch.signature = await sign(batch.signable())
+      batch.signature = await auth.sign(batch.signable())
       await this._truncate(batch, null)
     } finally {
       this.truncating--
@@ -214,19 +237,63 @@ module.exports = class Core {
     }
   }
 
-  async append (values, sign = this.defaultSign, hooks = {}) {
+  async clear (start, end) {
+    await this._mutex.lock()
+
+    const entry = {
+      userData: null,
+      treeNodes: null,
+      treeUpgrade: null,
+      bitfield: {
+        start,
+        length: end - start,
+        drop: true
+      }
+    }
+
+    await this.oplog.append([entry], false)
+
+    this.bitfield.setRange(start, end - start, false)
+
+    if (start < this.header.contiguousLength) {
+      this.header.contiguousLength = start
+    }
+
+    start = this.bitfield.lastSet(start) + 1
+    end = this.bitfield.firstSet(end)
+
+    if (end === -1) end = this.tree.length
+
+    try {
+      const offset = await this.tree.byteOffset(start * 2)
+      const [byteEnd, byteEndLength] = await this.tree.byteRange((end - 1) * 2)
+      const length = (byteEnd + byteEndLength) - offset
+
+      await this.blocks.clear(offset, length)
+
+      this.onupdate(0, entry.bitfield, null, null)
+
+      if (this._shouldFlush()) await this._flushOplog()
+    } finally {
+      this._mutex.unlock()
+    }
+  }
+
+  async append (values, auth = this.defaultAuth, hooks = {}) {
     await this._mutex.lock()
 
     try {
       if (hooks.preappend) await hooks.preappend(values)
 
-      if (!values.length) return this.tree.length
+      if (!values.length) {
+        return { length: this.tree.length, byteLength: this.tree.byteLength }
+      }
 
       const batch = this.tree.batch()
       for (const val of values) batch.append(val)
 
       const hash = batch.hash()
-      batch.signature = await sign(batch.signable(hash))
+      batch.signature = await auth.sign(this._legacy ? batch.signableLegacy(hash) : batch.signable(hash))
 
       const entry = {
         userData: null,
@@ -239,7 +306,8 @@ module.exports = class Core {
         }
       }
 
-      await this._appendBlocks(values)
+      const byteLength = await this._appendBlocks(values)
+
       await this.oplog.append([entry], false)
 
       this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
@@ -248,21 +316,28 @@ module.exports = class Core {
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, entry.bitfield, null, null)
+
+      const status = 0b0001 | updateContig(this.header, entry.bitfield, this.bitfield)
+      this.onupdate(status, entry.bitfield, null, null)
 
       if (this._shouldFlush()) await this._flushOplog()
 
-      return batch.ancestors
+      return { length: batch.length, byteLength }
     } finally {
       this._mutex.unlock()
     }
   }
 
+  _signed (batch, hash, auth = this.defaultAuth) {
+    const signable = this._legacy ? batch.signableLegacy(hash) : batch.signable(hash)
+    return auth.verify(signable, batch.signature)
+  }
+
   async _verifyExclusive ({ batch, bitfield, value, from }) {
     // TODO: move this to tree.js
     const hash = batch.hash()
-    if (!batch.signature || !this.crypto.verify(batch.signable(hash), batch.signature, this.header.signer.publicKey)) {
-      throw new Error('Remote signature does not match')
+    if (!batch.signature || !this._signed(batch, hash)) {
+      throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
     await this._mutex.lock()
@@ -281,14 +356,21 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      if (bitfield) this.bitfield.set(bitfield.start, true)
+      let status = 0b0001
+
+      if (bitfield) {
+        this.bitfield.set(bitfield.start, true)
+        status |= updateContig(this.header, bitfield, this.bitfield)
+      }
+
       batch.commit()
 
       this.header.tree.fork = batch.fork
       this.header.tree.length = batch.length
       this.header.tree.rootHash = batch.rootHash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, bitfield, value, from)
+
+      this.onupdate(status, bitfield, value, from)
 
       if (this._shouldFlush()) await this._flushOplog()
     } finally {
@@ -335,9 +417,16 @@ module.exports = class Core {
           continue
         }
 
-        if (bitfield) this.bitfield.set(bitfield.start, true)
+        let status = 0
+
+        if (bitfield) {
+          this.bitfield.set(bitfield.start, true)
+          status = updateContig(this.header, bitfield, this.bitfield)
+        }
+
         batch.commit()
-        this.onupdate(0, bitfield, value, from)
+
+        this.onupdate(status, bitfield, value, from)
       }
 
       if (this._shouldFlush()) await this._flushOplog()
@@ -415,13 +504,15 @@ module.exports = class Core {
     addReorgHint(this.header.hints.reorgs, this.tree, batch)
     batch.commit()
 
-    const appended = batch.length > batch.ancestors
+    const contigStatus = updateContig(this.header, entry.bitfield, this.bitfield)
+    const status = ((batch.length > batch.ancestors) ? 0b0011 : 0b0010) | contigStatus
 
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()
     this.header.tree.signature = batch.signature
-    this.onupdate(appended ? 0b11 : 0b10, entry.bitfield, null, from)
+
+    this.onupdate(status, entry.bitfield, null, from)
 
     // TODO: there is a bug in the merkle tree atm where it cannot handle unflushed
     // truncates if we append or download anything after the truncation point later on
@@ -442,6 +533,36 @@ module.exports = class Core {
   }
 }
 
+function updateContig (header, upd, bitfield) {
+  const end = upd.start + upd.length
+
+  let c = header.contiguousLength
+
+  if (upd.drop) {
+    // If we dropped a block in the current contig range, "downgrade" it
+    if (c <= end && c > upd.start) {
+      c = upd.start
+    }
+  } else {
+    if (c <= end && c >= upd.start) {
+      c = end
+      while (bitfield.get(c)) c++
+    }
+  }
+
+  if (c === header.contiguousLength) {
+    return 0b0000
+  }
+
+  if (c > header.contiguousLength) {
+    header.contiguousLength = c
+    return 0b0100
+  }
+
+  header.contiguousLength = c
+  return 0b1000
+}
+
 function addReorgHint (list, tree, batch) {
   if (tree.length === 0 || tree.fork === batch.fork) return
 
@@ -465,4 +586,25 @@ function updateUserData (list, key, value) {
   if (value) list.push({ key, value })
 }
 
+function closeAll (...storages) {
+  let missing = 1
+  let error = null
+
+  return new Promise((resolve, reject) => {
+    for (const s of storages) {
+      missing++
+      s.close(done)
+    }
+
+    done(null)
+
+    function done (err) {
+      if (err) error = err
+      if (--missing) return
+      if (error) reject(error)
+      else resolve()
+    }
+  })
+}
+
 function noop () {}

package/lib/download.js

@@ -0,0 +1,22 @@
+module.exports = class Download {
+  constructor (req) {
+    this.req = req
+  }
+
+  async done () {
+    return (await this.req).promise
+  }
+
+  /**
+   * Deprecated. Use `range.done()`.
+   */
+  downloaded () {
+    return this.done()
+  }
+
+  destroy () {
+    this.req.then(req => req.context && req.context.detach(req), noop)
+  }
+}
+
+function noop () {}

package/lib/errors.js

@@ -0,0 +1,50 @@
+module.exports = class HypercoreError extends Error {
+  constructor (msg, code, fn = HypercoreError) {
+    super(`${code}: ${msg}`)
+    this.code = code
+
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, fn)
+    }
+  }
+
+  get name () {
+    return 'HypercoreError'
+  }
+
+  static BAD_ARGUMENT (msg) {
+    return new HypercoreError(msg, 'BAD_ARGUMENT', HypercoreError.BAD_ARGUMENT)
+  }
+
+  static STORAGE_EMPTY (msg) {
+    return new HypercoreError(msg, 'STORAGE_EMPTY', HypercoreError.STORAGE_EMPTY)
+  }
+
+  static STORAGE_CONFLICT (msg) {
+    return new HypercoreError(msg, 'STORAGE_CONFLICT', HypercoreError.STORAGE_CONFLICT)
+  }
+
+  static INVALID_SIGNATURE (msg) {
+    return new HypercoreError(msg, 'INVALID_SIGNATURE', HypercoreError.INVALID_SIGNATURE)
+  }
+
+  static INVALID_CAPABILITY (msg) {
+    return new HypercoreError(msg, 'INVALID_CAPABILITY', HypercoreError.INVALID_CAPABILITY)
+  }
+
+  static SNAPSHOT_NOT_AVAILABLE (msg = 'Snapshot is not available') {
+    return new HypercoreError(msg, 'SNAPSHOT_NOT_AVAILABLE', HypercoreError.SNAPSHOT_NOT_AVAILABLE)
+  }
+
+  static REQUEST_CANCELLED (msg = 'Request was cancelled') {
+    return new HypercoreError(msg, 'REQUEST_CANCELLED', HypercoreError.REQUEST_CANCELLED)
+  }
+
+  static SESSION_NOT_WRITABLE (msg = 'Session is not writable') {
+    return new HypercoreError(msg, 'SESSION_NOT_WRITABLE', HypercoreError.SESSION_NOT_WRITABLE)
+  }
+
+  static SESSION_CLOSED (msg = 'Session is closed') {
+    return new HypercoreError(msg, 'SESSION_CLOSED', HypercoreError.SESSION_CLOSED)
+  }
+}

package/lib/info.js

@@ -0,0 +1,24 @@
+module.exports = class Info {
+  constructor (opts = {}) {
+    this.length = opts.length || 0
+    this.contiguousLength = opts.contiguousLength || 0
+    this.byteLength = opts.byteLength || 0
+    this.padding = opts.padding || 0
+  }
+
+  static async from (core, padding, snapshot) {
+    return new Info({
+      key: core.key,
+      discoveryKey: core.discoveryKey,
+      length: snapshot
+        ? snapshot.length
+        : core.tree.length,
+      contiguousLength: core.header.contiguousLength,
+      byteLength: snapshot
+        ? snapshot.byteLength
+        : (core.tree.byteLength - (core.tree.length * padding)),
+      fork: core.tree.fork,
+      padding
+    })
+  }
+}