hypercore 11.25.0 → 11.27.0

package/index.js

@@ -11,6 +11,9 @@ const safetyCatch = require('safety-catch')
 const unslab = require('unslab')
 const flat = require('flat-tree')
 
+const { SMALL_WANTS } = require('./lib/feature-flags')
+const { UPDATE_COMPAT } = require('./lib/wants')
+
 const inspect = require('./lib/inspect')
 const Core = require('./lib/core')
 const Info = require('./lib/info')
@@ -107,6 +110,13 @@ class Hypercore extends EventEmitter {
 
   static DefaultEncryption = DefaultEncryption
 
+  static SMALL_WANTS = SMALL_WANTS
+
+  static enable(flag) {
+    const enableCompat = (flag & SMALL_WANTS) === 0
+    UPDATE_COMPAT(enableCompat)
+  }
+
   static key(manifest, { compat, version, namespace } = {}) {
     if (b4a.isBuffer(manifest)) {
       manifest = { version, signers: [{ publicKey: manifest, namespace }] }
@@ -1047,22 +1057,27 @@ class Hypercore extends EventEmitter {
   async recoverFromRemoteProof(remoteProof) {
     this.core.replicator.setPushOnly(true)
     this.core._repairMode = true
+
     await this.core.state.mutex.lock()
-    const p = await verify(this.core.db, remoteProof)
-    if (!p) return false
 
-    const tx = this.core.storage.write()
-    for (const node of p.proof.upgrade.nodes) {
-      tx.putTreeNode(node)
-    }
-    await tx.flush()
+    try {
+      const p = await verify(this.core.db, remoteProof)
+      if (!p) return false
 
-    this.core.state.mutex.unlock()
-    const succeed = p.proof.upgrade.nodes.length !== 0
-    if (succeed) {
-      this.core.replicator.setPushOnly(false)
+      const tx = this.core.storage.write()
+      for (const node of p.proof.upgrade.nodes) {
+        tx.putTreeNode(node)
+      }
+      await tx.flush()
+
+      const succeed = p.proof.upgrade.nodes.length !== 0
+      if (succeed) {
+        this.core.replicator.setPushOnly(false)
+      }
+      return succeed
+    } finally {
+      this.core.state.mutex.unlock()
     }
-    return succeed
   }
 
   recoverTreeNodeFromPeers() {

package/lib/core.js

@@ -204,7 +204,8 @@ module.exports = class Core {
         hints: {
           reorgs: [],
           contiguousLength: 0,
-          remoteContiguousLength: 0
+          remoteContiguousLength: 0,
+          recovering: 0
         }
       }
 
@@ -553,7 +554,8 @@ module.exports = class Core {
 
     tx.setHints({
       contiguousLength: this.header.hints.contiguousLength,
-      remoteContiguousLength: this.header.hints.remoteContiguousLength
+      remoteContiguousLength: this.header.hints.remoteContiguousLength,
+      recovering: this.header.hints.recovering
     })
 
     await tx.flush()
@@ -963,7 +965,8 @@ function parseHeader(info) {
     hints: {
       reorgs: [],
       contiguousLength: info.hints ? info.hints.contiguousLength : 0,
-      remoteContiguousLength: info.hints ? info.hints.remoteContiguousLength : 0
+      remoteContiguousLength: info.hints ? info.hints.remoteContiguousLength : 0,
+      recovering: info.recovering || 0
     }
   }
 }

package/lib/feature-flags.js

@@ -0,0 +1,3 @@
+module.exports = {
+  SMALL_WANTS: 1
+}

package/lib/replicator.js

@@ -31,7 +31,8 @@ const ReceiverQueue = require('./receiver-queue')
 const HotswapQueue = require('./hotswap-queue')
 const RemoteBitfield = require('./remote-bitfield')
 const { MerkleTree } = require('./merkle-tree')
-const { LocalWants, RemoteWants, WANT_BATCH } = require('./wants')
+const w = require('./wants')
+const { LocalWants, RemoteWants } = w
 const {
   REQUEST_CANCELLED,
   REQUEST_TIMEOUT,
@@ -986,6 +987,7 @@ class Peer {
   async push(index) {
     if (!this.remoteAllowPush) return
     if (!this.remoteDownloading) return
+    if (this.core.state.length <= index) return
     if (this.core.state.fork !== this.remoteFork) return
     if (this.remoteBitfield.get(index)) return
 
@@ -1730,6 +1732,7 @@ class Peer {
   }
 
   _addRangeBatch(r, offset, end) {
+    const { WANT_BATCH } = w
     while (offset < end) {
       const next = this.core.bitfield.findFirst(false, offset)
       if (next === -1 || next >= end) return false
@@ -1745,6 +1748,7 @@ class Peer {
   }
 
   _populateRangeBatches(r) {
+    const { WANT_BATCH } = w
     for (let i = r.batches.length - 1; i >= 0; i--) {
       const b = r.batches[i]
       const minStart = b.batch * WANT_BATCH
@@ -1800,6 +1804,7 @@ class Peer {
   }
 
   _requestRange(r) {
+    const { WANT_BATCH } = w
     if (this.syncsProcessing > 0) return false
 
     const { length, fork } = this.core.state

package/lib/session-state.js

@@ -4,7 +4,13 @@ const assert = require('nanoassert')
 const flat = require('flat-tree')
 const quickbit = require('quickbit-universal')
 
-const { INVALID_OPERATION, INVALID_SIGNATURE, ASSERTION } = require('hypercore-errors')
+const {
+  INVALID_OPERATION,
+  INVALID_SIGNATURE,
+  ASSERTION,
+  SESSION_CLOSED,
+  SESSION_NOT_WRITABLE
+} = require('hypercore-errors')
 
 const Mutex = require('./mutex')
 const Bitfield = require('./bitfield')
@@ -562,9 +568,47 @@ module.exports = class SessionState {
     }
   }
 
+  // simple tool to wait for other peers to catch us up
+  async _waitForRecovery() {
+    const length = this.length
+    const replicator = this.core.replicator
+
+    while (!this.closing) {
+      if (replicator.peers.length < 1) {
+        await new Promise((resolve) => setTimeout(resolve, 5000))
+        continue
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 5000))
+
+      const peers = replicator.peers
+
+      if (peers.length < 1 || this.closing) continue
+      if (peers[0].remoteLength > this.length) continue
+
+      if (this.length !== length) {
+        throw SESSION_NOT_WRITABLE('core recovered during append')
+      }
+
+      this.core.header.hints.recovering = 0
+
+      await this.mutex.lock()
+      try {
+        await this.core.flushHints()
+        return
+      } finally {
+        this.mutex.unlock()
+      }
+    }
+
+    throw SESSION_CLOSED('Cannot recover a closed session', this.discoveryKey)
+  }
+
   async append(values, { signature, keyPair, preappend, postappend, maxLength = -1 } = {}) {
     if (!keyPair && this.isDefault()) keyPair = this.core.header.keyPair
 
+    if (this.isDefault() && this.core.header.hints.recovering) await this._waitForRecovery()
+
     await this.mutex.lock()
 
     try {
@@ -921,6 +965,8 @@ module.exports = class SessionState {
       throw ASSERTION('Cannot commit while repair mode is on')
     }
 
+    if (this.core.header.hints.recovering) await this._waitForRecovery()
+
     let srcLocked = false
     await this.mutex.lock()
 

package/lib/wants.js

@@ -1,7 +1,7 @@
-const COMPAT = true // should be flipped once this is widely deployed
-const BATCH_UINTS = COMPAT ? 65536 : 128
-const BATCH_BYTES = BATCH_UINTS * 4
-const BATCH = BATCH_BYTES * 8 // in bits
+let COMPAT = true
+let BATCH_UINTS = COMPAT ? 65536 : 128
+let BATCH_BYTES = BATCH_UINTS * 4
+let BATCH = BATCH_BYTES * 8 // in bits
 const MAX_REMOTE_BATCHES = 4
 const MAX_RANGE = 8 * 256 * 1024
 const MIN_RANGE = 32 // 4bit ints
@@ -295,9 +295,18 @@ class RemoteWants {
   }
 }
 
+function UPDATE_COMPAT(isCompat) {
+  COMPAT = isCompat
+  BATCH_UINTS = COMPAT ? 65536 : 128
+  BATCH_BYTES = BATCH_UINTS * 4
+  BATCH = BATCH_BYTES * 8 // in bits
+  exports.WANT_BATCH = BATCH
+}
+
 exports.LocalWants = LocalWants
 exports.RemoteWants = RemoteWants
 exports.WANT_BATCH = BATCH
+exports.UPDATE_COMPAT = UPDATE_COMPAT
 
 function validateBatchRange(range) {
   if (range.length > MAX_RANGE || range.length < MIN_RANGE) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.25.0",
+  "version": "11.27.0",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {