autobee 2.11.7 → 2.12.0

package/README.md

@@ -285,10 +285,12 @@ Pass one of `head` or `legacy`, not both.
 
 #### `fastForward.conservative`
 
-Defaults to `true`: only fast-forward once the system and view lengths the head stamps are held whole, either locally or by a connected peer.
+Defaults to `false`. When `true`, only fast-forward once the system and view lengths the head stamps are held whole, either locally or by a connected peer.
 
 The check runs at the end of the fast-forward, after the system has booted and the view head has been fetched, so a sparse view nobody can serve is skipped instead of landed on.
 
+A node that landed via fast-forward holds its cores sparse, so it never satisfies the check for others. Leave it off unless every peer is warmed up to hold the view whole.
+
 #### `await db.moveTo(head, [options])`
 
 Fast-forward onto an oplog head, ignoring the usual distance and conservative checks. Resolves `{ to, from }`, or `null` if the head could not be booted.

package/index.js

@@ -110,8 +110,8 @@ module.exports = class Autobee extends ReadyResource {
     // oplog head to boot from: migrates or fast-forwards depending on its version
     this.bootFrom = (fastForward && fastForward.boot) || null
 
-    // conservative (default on): only fast-forward onto a system and view held whole
-    this._conservativeFF = fastForward === null || fastForward.conservative !== false
+    // conservative (default off): only fast-forward onto a system and view held whole
+    this._conservativeFF = fastForward !== null && fastForward.conservative === true
 
     this.trusted = new TrustedPeers(this, handlers)
 
@@ -987,15 +987,7 @@ module.exports = class Autobee extends ReadyResource {
     for (const res of ops) {
       if (res === null) continue
 
-      // the head we were woken on is a candidate in its own right
       heads.push({ key: res.key, length: res.length })
-
-      if (!res.op.trusted) continue
-
-      for (const trusted of res.op.trusted) {
-        const head = this.trusted.read(trusted)
-        if (head !== null) heads.push(head)
-      }
     }
 
     return heads
@@ -1612,10 +1604,11 @@ module.exports = class Autobee extends ReadyResource {
     if (!this._bootGuard.opened) await this._bootGuard.ready()
     if (this.closing) throw new Error('Autobee closed')
 
-    const ff = await FastForward.fromHead(this, head, null, { force: true, timeout })
+    // an explicit move goes wherever it is pointed, backwards included
+    const ff = await FastForward.fromHead(this, head, null, { force: true, rewind: true, timeout })
     if (ff === null) return null
 
-    if (!(await this._runFastForward(ff, { force: true }))) return null
+    if (!(await this._runFastForward(ff, { force: true, rewind: true }))) return null
 
     return this.ff.promise
   }
@@ -1681,7 +1674,7 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _runFastForward(ff, { force = false } = {}) {
+  async _runFastForward(ff, { force = false, rewind = false } = {}) {
     if (this.fastForwardTo !== null || this.fastForwarding !== null) {
       await ff.close()
       return false
@@ -1697,6 +1690,9 @@ module.exports = class Autobee extends ReadyResource {
 
     if (!result) return false
 
+    // never move backwards, a boot head can be older than what we already have
+    if (!rewind && flushes < this.system.flushes) return false
+
     // apply ran while we searched, only drop it if it no longer moves us forward
     if (!force && flushes <= this.system.flushes) return false
 

package/lib/fast-forward.js

@@ -39,17 +39,33 @@ class FastForward {
 
   static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
 
-  static async fromHead(auto, head, trusted, { force = false, skipGap = false, timeout = 0 } = {}) {
-    const conservative = !force && auto._conservativeFF
-
+  static async fromHead(
+    auto,
+    head,
+    trusted,
+    { force = false, skipGap = false, rewind = false, timeout = 0 } = {}
+  ) {
     const oplog = await FastForward.flushHead(auto, head, { timeout })
     if (!oplog) return null
 
     const verified = trusted ? await FastForward.flushHead(auto, trusted, { timeout }) : oplog
     if (!verified) return null
 
+    return FastForward.fromVerified(auto, verified, { force, skipGap, rewind, timeout })
+  }
+
+  static fromVerified(
+    auto,
+    verified,
+    { force = false, skipGap = false, rewind = false, timeout = 0 } = {}
+  ) {
+    const conservative = !force && auto._conservativeFF
+
     // legacy nodes from non-indexers have no system info to fast-forward from
-    if (!oplog.op.views || !verified.op.views) return null
+    if (!verified.op.views) return null
+
+    // never move backwards, even when forced or the gap is skipped
+    if (!rewind && verified.op.views.flushes < auto.system.flushes) return null
 
     if (!force && !skipGap && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
       return null
@@ -63,68 +79,95 @@ class FastForward {
 
   static async fromHeads(auto, heads, { force = false, skipGap = false, timeout = 0 } = {}) {
     const reference = auto._workingView.view
+    const opTimeout = timeout || DEFAULT_OP_TIMEOUT
 
-    const promises = []
-
-    for (const head of heads) {
-      if (head.length === 0) continue
-      promises.push(FastForward.flushHead(auto, head, { timeout }))
-    }
-
-    const ops = await FastForward.reindexedCandidates(auto, await Promise.all(promises), {
-      timeout: timeout || DEFAULT_OP_TIMEOUT
-    })
+    const hints = await FastForward.flushHeads(auto, heads, timeout)
     if (auto.fastForwarding || auto.fastForwardTo) return null
 
-    const trust = await Promise.all(ops.map((res) => auto.trusted.isTrusted(res.key, reference)))
+    const trust = await Promise.all(hints.map((res) => auto.trusted.isTrusted(res.key, reference)))
     if (auto.fastForwarding || auto.fastForwardTo) return null
 
-    const candidates = []
+    const landings = new Map()
+    const unverified = new Map()
 
-    let bestTrusted = null
-    let bestTrustedFlushes = -1
-
-    for (let i = 0; i < ops.length; i++) {
-      const res = ops[i]
-      if (!force && !skipGap && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
+    const promises = []
 
-      if (!trust[i]) {
-        candidates.push(res)
+    for (let i = 0; i < hints.length; i++) {
+      if (trust[i]) {
+        addOplog(landings, hints[i])
         continue
       }
 
-      if (res.op.views.flushes > bestTrustedFlushes) {
-        bestTrustedFlushes = res.op.views.flushes
-        bestTrusted = res
-      }
+      promises.push(
+        FastForward.signalled(auto, hints[i], reference, timeout).then((sig) => {
+          for (const { oplog, trusted } of sig) addOplog(trusted ? landings : unverified, oplog)
+        })
+      )
     }
 
-    if (bestTrusted !== null) {
-      return FastForward.fromHead(auto, bestTrusted, null, { force, skipGap, timeout })
+    await Promise.all(promises)
+    if (auto.fastForwarding || auto.fastForwardTo) return null
+
+    const ranked = [...landings.values()].sort(byFlushes)
+
+    const reindexed = ranked.map((res) => FastForward.isReindexed(auto, res, opTimeout))
+
+    for (let i = 0; i < ranked.length; i++) {
+      if (!(await reindexed[i])) continue
+      if (auto.fastForwarding || auto.fastForwardTo) return null
+
+      const ff = FastForward.fromVerified(auto, ranked[i], { force, skipGap, timeout })
+      if (ff !== null) return ff
     }
 
-    candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
+    for (const sig of [...unverified.values()].sort(byFlushes)) {
+      if (sig.op.views.flushes < auto.system.flushes) continue
+      if (!force && !skipGap && sig.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
 
-    for (const res of candidates) {
-      const trusted = await FastForward.mostRecentTrusted(auto, res, reference, timeout)
-      if (trusted === null) continue
+      const head = await FastForward.mostRecentTrusted(auto, sig, reference, timeout)
+      if (head === null) continue
       if (auto.fastForwarding || auto.fastForwardTo) return null
 
-      const ff = await FastForward.fromHead(auto, res, trusted, { force, skipGap, timeout })
+      const verified = await FastForward.flushHead(auto, head, { timeout })
+      if (verified === null) continue
+      if (!(await FastForward.isReindexed(auto, verified, opTimeout))) continue
+      if (auto.fastForwarding || auto.fastForwardTo) return null
+
+      const ff = FastForward.fromVerified(auto, verified, { force, skipGap, timeout })
       if (ff !== null) return ff
     }
 
     return null
   }
 
-  static async reindexedCandidates(auto, ops, { timeout }) {
-    const withViews = ops.filter((res) => res !== null && res.op.views)
-    const reindexed = await Promise.all(
-      withViews.map((res) => FastForward.isReindexed(auto, res, timeout))
+  static async flushHeads(auto, heads, timeout) {
+    const promises = []
+
+    for (const head of heads) {
+      if (head.length === 0) continue
+      promises.push(FastForward.flushHead(auto, head, { timeout }))
+    }
+
+    return (await Promise.all(promises)).filter((res) => res !== null)
+  }
+
+  static async signalled(auto, res, reference, timeout) {
+    const heads = []
+
+    for (const entry of res.op.trusted || []) {
+      const head = auto.trusted.read(entry)
+      if (head !== null && head.length > 0) heads.push(head)
+    }
+
+    const oplogs = await FastForward.flushHeads(auto, heads, timeout)
+    const trust = await Promise.all(
+      oplogs.map((oplog) => auto.trusted.isTrusted(oplog.key, reference))
     )
-    return withViews.filter((res, i) => reindexed[i])
+
+    return oplogs.map((oplog, i) => ({ oplog, trusted: !!trust[i] }))
   }
 
+  // never rejects, the search may return before awaiting every check
   static async isReindexed(auto, res, timeout) {
     const { system, view } = res.op.views
     const checks = [system, view]
@@ -135,8 +178,13 @@ class FastForward {
         return auto._shouldReindex(key, { unknown: true, length, timeout })
       })
 
-    for (const shouldReindex of await Promise.all(checks)) {
-      if (shouldReindex) return false
+    try {
+      for (const shouldReindex of await Promise.all(checks)) {
+        if (shouldReindex) return false
+      }
+    } catch (err) {
+      safetyCatch(err)
+      return false
     }
 
     return true
@@ -395,6 +443,14 @@ function isContiguous(core, length) {
   return core.contiguousLength >= length || core.remoteContiguousLength >= length
 }
 
+function byFlushes(a, b) {
+  return b.op.views.flushes - a.op.views.flushes
+}
+
+function addOplog(map, oplog) {
+  map.set(b4a.toString(oplog.key, 'hex') + ':' + oplog.length, oplog)
+}
+
 function batchToHead(b) {
   return {
     key: b.key,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.11.7",
+  "version": "2.12.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",