hypercore 10.0.0-alpha.51 → 10.0.0-alpha.54

package/index.js

@@ -1,5 +1,5 @@
 const { EventEmitter } = require('events')
-const raf = require('random-access-file')
+const RAF = require('random-access-file')
 const isOptions = require('is-options')
 const hypercoreCrypto = require('hypercore-crypto')
 const c = require('compact-encoding')
@@ -9,8 +9,6 @@ const NoiseSecretStream = require('@hyperswarm/secret-stream')
 const Protomux = require('protomux')
 const codecs = require('codecs')
 
-const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
-
 const Replicator = require('./lib/replicator')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
@@ -162,14 +160,25 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   static defaultStorage (storage, opts = {}) {
-    if (typeof storage !== 'string') return storage
+    if (typeof storage !== 'string') {
+      if (!isRandomAccessClass(storage)) return storage
+      const Cls = storage // just to satisfy standard...
+      return name => new Cls(name)
+    }
+
     const directory = storage
     const toLock = opts.lock || 'oplog'
-    return function createFile (name) {
-      const locked = name === toLock || name.endsWith('/' + toLock)
-      const lock = locked ? fsctl.lock : null
-      const sparse = locked ? null : null // fsctl.sparse, disable sparse on windows - seems to fail for some people. TODO: investigate
-      return raf(name, { directory, lock, sparse })
+
+    return createFile
+
+    function createFile (name) {
+      const lock = isFile(name, toLock)
+      const sparse = isFile(name, 'data') || isFile(name, 'bitfield') || isFile(name, 'tree')
+      return new RAF(name, { directory, lock, sparse })
+    }
+
+    function isFile (name, n) {
+      return name === n || name.endsWith('/' + n)
     }
   }
 
@@ -303,8 +312,7 @@ module.exports = class Hypercore extends EventEmitter {
       crypto: this.crypto,
       legacy: opts.legacy,
       auth: opts.auth,
-      onupdate: this._oncoreupdate.bind(this),
-      oncontigupdate: this._oncorecontigupdate.bind(this)
+      onupdate: this._oncoreupdate.bind(this)
     })
 
     if (opts.userData) {
@@ -487,30 +495,38 @@ module.exports = class Hypercore extends EventEmitter {
 
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
-      const truncated = (status & 0b10) !== 0
-      const appended = (status & 0b01) !== 0
+      const truncatedNonSparse = (status & 0b1000) !== 0
+      const appendedNonSparse = (status & 0b0100) !== 0
+      const truncated = (status & 0b0010) !== 0
+      const appended = (status & 0b0001) !== 0
 
       if (truncated) {
         this.replicator.ontruncate(bitfield.start)
       }
 
+      if ((status & 0b0011) !== 0) {
+        this.replicator.onupgrade()
+      }
+
       for (let i = 0; i < this.sessions.length; i++) {
         const s = this.sessions[i]
 
         if (truncated) {
           if (s.cache) s.cache.clear()
-          s.emit('truncate', bitfield.start, this.core.tree.fork)
+
           // If snapshotted, make sure to update our compat so we can fail gets
           if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
         }
 
-        // For sparse sessions, immediately emit appends. Non-sparse sessions
-        // are handled separately and only emit appends when their contiguous
-        // length is updated.
-        if (appended && s.sparse) s.emit('append')
-      }
+        if (s.sparse ? truncated : truncatedNonSparse) {
+          s.emit('truncate', bitfield.start, this.core.tree.fork)
+        }
 
-      this.replicator.onupgrade()
+        // For sparse sessions, immediately emit appends. If non-sparse, emit if contig length has updated
+        if (s.sparse ? appended : appendedNonSparse) {
+          s.emit('append')
+        }
+      }
     }
 
     if (bitfield) {
@@ -526,16 +542,6 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  _oncorecontigupdate () {
-    // For non-sparse sessions, emit appends only when the contiguous length is
-    // updated.
-    for (let i = 0; i < this.sessions.length; i++) {
-      const s = this.sessions[i]
-
-      if (!s.sparse) s.emit('append')
-    }
-  }
-
   _onpeerupdate (added, peer) {
     const name = added ? 'peer-add' : 'peer-remove'
 
@@ -852,12 +858,8 @@ function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
 
-function requireMaybe (name) {
-  try {
-    return require(name)
-  } catch (_) {
-    return null
-  }
+function isRandomAccessClass (fn) {
+  return !!(typeof fn === 'function' && fn.prototype && typeof fn.prototype.open === 'function')
 }
 
 function toHex (buf) {

package/lib/core.js

@@ -9,9 +9,8 @@ const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = req
 const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate, oncontigupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
     this.onupdate = onupdate
-    this.oncontigupdate = oncontigupdate
     this.header = header
     this.crypto = crypto
     this.oplog = oplog
@@ -27,8 +26,6 @@ module.exports = class Core {
     this._verifiesFlushed = null
     this._mutex = new Mutex()
     this._legacy = legacy
-
-    this._updateContiguousLength(header.contiguousLength)
   }
 
   static async open (storage, opts = {}) {
@@ -134,6 +131,11 @@ module.exports = class Core {
       await bitfield.clear()
     }
 
+    // compat from earlier version that do not store contig length
+    if (header.contiguousLength === 0) {
+      while (bitfield.get(header.contiguousLength)) header.contiguousLength++
+    }
+
     const auth = opts.auth || this.createAuth(crypto, header.signer)
 
     for (const e of entries) {
@@ -149,6 +151,7 @@ module.exports = class Core {
 
       if (e.bitfield) {
         bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
+        updateContig(header, e.bitfield, bitfield)
       }
 
       if (e.treeUpgrade) {
@@ -165,7 +168,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.oncontigupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
   }
 
   _shouldFlush () {
@@ -196,17 +199,6 @@ module.exports = class Core {
     await this.blocks.put(index, value, byteOffset)
   }
 
-  _updateContiguousLength (index, length = 0) {
-    if (index === this.header.contiguousLength) {
-      let i = this.header.contiguousLength + length
-
-      while (this.bitfield.get(i)) i++
-
-      this.header.contiguousLength = i
-      this.oncontigupdate()
-    }
-  }
-
   async userData (key, value) {
     // TODO: each oplog append can set user data, so we should have a way
     // to just hitch a ride on one of the other ongoing appends?
@@ -286,12 +278,12 @@ module.exports = class Core {
       this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
       batch.commit()
 
-      this.header.contiguousLength = batch.length
-      this.oncontigupdate()
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, entry.bitfield, null, null)
+
+      const status = 0b0001 | updateContig(this.header, entry.bitfield, this.bitfield)
+      this.onupdate(status, entry.bitfield, null, null)
 
       if (this._shouldFlush()) await this._flushOplog()
 
@@ -329,9 +321,11 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
+      let status = 0b0001
+
       if (bitfield) {
         this.bitfield.set(bitfield.start, true)
-        this._updateContiguousLength(bitfield.start, bitfield.length)
+        status |= updateContig(this.header, bitfield, this.bitfield)
       }
 
       batch.commit()
@@ -340,7 +334,8 @@ module.exports = class Core {
       this.header.tree.length = batch.length
       this.header.tree.rootHash = batch.rootHash
       this.header.tree.signature = batch.signature
-      this.onupdate(0b01, bitfield, value, from)
+
+      this.onupdate(status, bitfield, value, from)
 
       if (this._shouldFlush()) await this._flushOplog()
     } finally {
@@ -387,14 +382,16 @@ module.exports = class Core {
           continue
         }
 
+        let status = 0
+
         if (bitfield) {
           this.bitfield.set(bitfield.start, true)
-          this._updateContiguousLength(bitfield.start, bitfield.length)
+          status = updateContig(this.header, bitfield, this.bitfield)
         }
 
         batch.commit()
 
-        this.onupdate(0, bitfield, value, from)
+        this.onupdate(status, bitfield, value, from)
       }
 
       if (this._shouldFlush()) await this._flushOplog()
@@ -472,15 +469,15 @@ module.exports = class Core {
     addReorgHint(this.header.hints.reorgs, this.tree, batch)
     batch.commit()
 
-    const appended = batch.length > batch.ancestors
+    const contigStatus = updateContig(this.header, entry.bitfield, this.bitfield)
+    const status = ((batch.length > batch.ancestors) ? 0b0011 : 0b0010) | contigStatus
 
-    this.header.contiguousLength = Math.min(batch.ancestors, this.header.contiguousLength)
-    this.oncontigupdate()
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()
     this.header.tree.signature = batch.signature
-    this.onupdate(appended ? 0b11 : 0b10, entry.bitfield, null, from)
+
+    this.onupdate(status, entry.bitfield, null, from)
 
     // TODO: there is a bug in the merkle tree atm where it cannot handle unflushed
     // truncates if we append or download anything after the truncation point later on
@@ -501,6 +498,36 @@ module.exports = class Core {
   }
 }
 
+function updateContig (header, upd, bitfield) {
+  const end = upd.start + upd.length
+
+  let c = header.contiguousLength
+
+  if (upd.drop) {
+    // If we dropped a block in the current contig range, "downgrade" it
+    if (c <= end && c > upd.start) {
+      c = upd.start
+    }
+  } else {
+    if (c <= end && c >= upd.start) {
+      c = end
+      while (bitfield.get(c)) c++
+    }
+  }
+
+  if (c === header.contiguousLength) {
+    return 0b0000
+  }
+
+  if (c > header.contiguousLength) {
+    header.contiguousLength = c
+    return 0b0100
+  }
+
+  header.contiguousLength = c
+  return 0b1000
+}
+
 function addReorgHint (list, tree, batch) {
   if (tree.length === 0 || tree.fork === batch.fork) return
 

package/lib/merkle-tree.js

@@ -1,11 +1,13 @@
 const flat = require('flat-tree')
 const crypto = require('hypercore-crypto')
 const c = require('compact-encoding')
+const Xache = require('xache')
 const b4a = require('b4a')
 const caps = require('./caps')
 
 const BLANK_HASH = b4a.alloc(32)
 const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
+const TREE_CACHE = 128 // speeds up linear scans by A LOT
 
 class NodeQueue {
   constructor (nodes, extra = null) {
@@ -140,6 +142,7 @@ class MerkleTreeBatch {
         : this.ancestors
 
       this.tree.truncated = true
+      this.tree.cache = new Xache({ maxSize: this.tree.cache.maxSize })
       truncateMap(this.tree.unflushed, this.ancestors)
       if (this.tree.flushing !== null) truncateMap(this.tree.flushing, this.ancestors)
     }
@@ -350,6 +353,7 @@ module.exports = class MerkleTree {
 
     this.storage = storage
     this.unflushed = new Map()
+    this.cache = new Xache({ maxSize: TREE_CACHE })
     this.flushing = null
     this.truncated = false
     this.truncateTo = 0
@@ -403,6 +407,9 @@ module.exports = class MerkleTree {
   }
 
   get (index, error = true) {
+    const c = this.cache.get(index)
+    if (c) return c
+
     let node = this.unflushed.get(index)
 
     if (this.flushing !== null && node === undefined) {
@@ -422,7 +429,7 @@ module.exports = class MerkleTree {
       return Promise.resolve(node)
     }
 
-    return getStoredNode(this.storage, index, error)
+    return getStoredNode(this.storage, index, this.cache, error)
   }
 
   async flush () {
@@ -495,6 +502,7 @@ module.exports = class MerkleTree {
   }
 
   clear () {
+    this.cache = new Xache({ maxSize: this.cache.maxSize })
     this.truncated = true
     this.truncateTo = 0
     this.roots = []
@@ -754,7 +762,7 @@ module.exports = class MerkleTree {
 
     const roots = []
     for (const index of flat.fullRoots(2 * length)) {
-      roots.push(await getStoredNode(storage, index, true))
+      roots.push(await getStoredNode(storage, index, null, true))
     }
 
     return new MerkleTree(storage, roots, opts.fork || 0, opts.signature || null)
@@ -1085,7 +1093,7 @@ function blankNode (index) {
 
 // Storage methods
 
-function getStoredNode (storage, index, error) {
+function getStoredNode (storage, index, cache, error) {
   return new Promise((resolve, reject) => {
     storage.read(40 * index, 40, (err, data) => {
       if (err) {
@@ -1103,7 +1111,9 @@ function getStoredNode (storage, index, error) {
         return
       }
 
-      resolve({ index, size, hash })
+      const node = { index, size, hash }
+      if (cache !== null) cache.set(index, node)
+      resolve(node)
     })
   })
 }
@@ -1122,7 +1132,7 @@ async function autoLength (storage) {
   if (!nodes) return 0
   const ite = flat.iterator(nodes - 1)
   let index = nodes - 1
-  while (await getStoredNode(storage, ite.parent(), false)) index = ite.index
+  while (await getStoredNode(storage, ite.parent(), null, false)) index = ite.index
   return flat.rightSpan(index) / 2 + 1
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.51",
+  "version": "10.0.0-alpha.54",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -43,22 +43,19 @@
     "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
     "protomux": "^3.2.0",
-    "random-access-file": "^2.1.4",
+    "random-access-file": "^3.0.1",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
     "sodium-universal": "^3.0.4",
     "streamx": "^2.12.4",
-    "xache": "^1.0.0"
+    "xache": "^1.1.0"
   },
   "devDependencies": {
     "brittle": "^2.0.0",
-    "hyperswarm": "next",
-    "random-access-memory": "^4.1.0",
-    "random-access-memory-overlay": "^1.0.0",
+    "hyperswarm": "^4.1.1",
+    "random-access-memory": "^5.0.0",
+    "random-access-memory-overlay": "^2.0.0",
     "standard": "^16.0.3",
     "tmp-promise": "^3.0.2"
-  },
-  "optionalDependencies": {
-    "fsctl": "^1.0.0"
   }
 }