hypercore 10.0.0-alpha.50 → 10.0.0-alpha.53

package/index.js

@@ -1,5 +1,5 @@
 const { EventEmitter } = require('events')
-const raf = require('random-access-file')
+const RAF = require('random-access-file')
 const isOptions = require('is-options')
 const hypercoreCrypto = require('hypercore-crypto')
 const c = require('compact-encoding')
@@ -9,8 +9,6 @@ const NoiseSecretStream = require('@hyperswarm/secret-stream')
 const Protomux = require('protomux')
 const codecs = require('codecs')
 
-const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
-
 const Replicator = require('./lib/replicator')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
@@ -162,14 +160,25 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   static defaultStorage (storage, opts = {}) {
-    if (typeof storage !== 'string') return storage
+    if (typeof storage !== 'string') {
+      if (!isRandomAccessClass(storage)) return storage
+      const Cls = storage // just to satisfy standard...
+      return name => new Cls(name)
+    }
+
     const directory = storage
     const toLock = opts.lock || 'oplog'
-    return function createFile (name) {
-      const locked = name === toLock || name.endsWith('/' + toLock)
-      const lock = locked ? fsctl.lock : null
-      const sparse = locked ? null : null // fsctl.sparse, disable sparse on windows - seems to fail for some people. TODO: investigate
-      return raf(name, { directory, lock, sparse })
+
+    return createFile
+
+    function createFile (name) {
+      const lock = isFile(name, toLock)
+      const sparse = isFile(name, 'data') || isFile(name, 'bitfield') || isFile(name, 'tree')
+      return new RAF(name, { directory, lock, sparse })
+    }
+
+    function isFile (name, n) {
+      return name === n || name.endsWith('/' + n)
     }
   }
 
@@ -852,12 +861,8 @@ function isStream (s) {
   return typeof s === 'object' && s && typeof s.pipe === 'function'
 }
 
-function requireMaybe (name) {
-  try {
-    return require(name)
-  } catch (_) {
-    return null
-  }
+function isRandomAccessClass (fn) {
+  return !!(typeof fn === 'function' && fn.prototype && typeof fn.prototype.open === 'function')
 }
 
 function toHex (buf) {

package/lib/merkle-tree.js

@@ -1,11 +1,13 @@
 const flat = require('flat-tree')
 const crypto = require('hypercore-crypto')
 const c = require('compact-encoding')
+const Xache = require('xache')
 const b4a = require('b4a')
 const caps = require('./caps')
 
 const BLANK_HASH = b4a.alloc(32)
 const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
+const TREE_CACHE = 128 // speeds up linear scans by A LOT
 
 class NodeQueue {
   constructor (nodes, extra = null) {
@@ -140,6 +142,7 @@ class MerkleTreeBatch {
         : this.ancestors
 
       this.tree.truncated = true
+      this.tree.cache = new Xache({ maxSize: this.tree.cache.maxSize })
       truncateMap(this.tree.unflushed, this.ancestors)
       if (this.tree.flushing !== null) truncateMap(this.tree.flushing, this.ancestors)
     }
@@ -350,6 +353,7 @@ module.exports = class MerkleTree {
 
     this.storage = storage
     this.unflushed = new Map()
+    this.cache = new Xache({ maxSize: TREE_CACHE })
     this.flushing = null
     this.truncated = false
     this.truncateTo = 0
@@ -403,6 +407,9 @@ module.exports = class MerkleTree {
   }
 
   get (index, error = true) {
+    const c = this.cache.get(index)
+    if (c) return c
+
     let node = this.unflushed.get(index)
 
     if (this.flushing !== null && node === undefined) {
@@ -422,7 +429,7 @@ module.exports = class MerkleTree {
       return Promise.resolve(node)
     }
 
-    return getStoredNode(this.storage, index, error)
+    return getStoredNode(this.storage, index, this.cache, error)
   }
 
   async flush () {
@@ -495,6 +502,7 @@ module.exports = class MerkleTree {
   }
 
   clear () {
+    this.cache = new Xache({ maxSize: this.cache.maxSize })
     this.truncated = true
     this.truncateTo = 0
     this.roots = []
@@ -754,7 +762,7 @@ module.exports = class MerkleTree {
 
     const roots = []
     for (const index of flat.fullRoots(2 * length)) {
-      roots.push(await getStoredNode(storage, index, true))
+      roots.push(await getStoredNode(storage, index, null, true))
     }
 
     return new MerkleTree(storage, roots, opts.fork || 0, opts.signature || null)
@@ -1085,7 +1093,7 @@ function blankNode (index) {
 
 // Storage methods
 
-function getStoredNode (storage, index, error) {
+function getStoredNode (storage, index, cache, error) {
   return new Promise((resolve, reject) => {
     storage.read(40 * index, 40, (err, data) => {
       if (err) {
@@ -1103,7 +1111,9 @@ function getStoredNode (storage, index, error) {
         return
       }
 
-      resolve({ index, size, hash })
+      const node = { index, size, hash }
+      if (cache !== null) cache.set(index, node)
+      resolve(node)
     })
   })
 }
@@ -1122,7 +1132,7 @@ async function autoLength (storage) {
   if (!nodes) return 0
   const ite = flat.iterator(nodes - 1)
   let index = nodes - 1
-  while (await getStoredNode(storage, ite.parent(), false)) index = ite.index
+  while (await getStoredNode(storage, ite.parent(), null, false)) index = ite.index
   return flat.rightSpan(index) / 2 + 1
 }
 

package/lib/replicator.js

@@ -1191,7 +1191,7 @@ module.exports = class Replicator {
       for (let i = 0; i < this._ranges.length; i++) {
         const r = this._ranges[i]
 
-        while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.start)) === true) r.start++
+        while ((r.end === -1 || r.start < r.end) && this.core.bitfield.get(mapIndex(r.blocks, r.start)) === true) r.start++
         while (r.start < r.end && this.core.bitfield.get(mapIndex(r.blocks, r.end - 1)) === true) r.end--
 
         if (r.end !== -1 && r.start >= r.end) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.50",
+  "version": "10.0.0-alpha.53",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -43,22 +43,19 @@
     "hypercore-crypto": "^3.2.1",
     "is-options": "^1.0.1",
     "protomux": "^3.2.0",
-    "random-access-file": "^2.1.4",
+    "random-access-file": "^3.0.1",
     "random-array-iterator": "^1.0.0",
     "safety-catch": "^1.0.1",
     "sodium-universal": "^3.0.4",
     "streamx": "^2.12.4",
-    "xache": "^1.0.0"
+    "xache": "^1.1.0"
   },
   "devDependencies": {
     "brittle": "^2.0.0",
-    "hyperswarm": "next",
-    "random-access-memory": "^4.1.0",
-    "random-access-memory-overlay": "^1.0.0",
+    "hyperswarm": "^4.1.1",
+    "random-access-memory": "^5.0.0",
+    "random-access-memory-overlay": "^2.0.0",
     "standard": "^16.0.3",
     "tmp-promise": "^3.0.2"
-  },
-  "optionalDependencies": {
-    "fsctl": "^1.0.0"
   }
 }