autobee 2.0.0-rc.15 → 2.0.0-rc.16

package/README.md

@@ -78,7 +78,10 @@ Options:
   encryptionKey: Buffer,         // 32-byte key to encrypt all data at rest
   encrypted: false,              // set true if using encryptionKey
   keyPair: { publicKey, secretKey }, // custom signing key pair for the local writer
-  optimistic: true               // allow optimistic writes from unknown writers
+  optimistic: true,              // allow optimistic writes from unknown writers
+  isTrusted (key, reference) {}, // do we trust this writer, see Fast-forward
+  mostRecentTrusted (target, reference) {}, // the head we vouch for, see Fast-forward
+  fastForward: {}                // see Fast-forward
 }
 ```
 
@@ -222,6 +225,50 @@ Create an anchor node. Returns `{ key, length }`. Anchors are used to create a v
 
 `true` if the system has not yet processed any nodes. Use this to bootstrap the first writer.
 
+### Fast-forward
+
+Fast-forward only ever deals in oplog heads — `{ key, length }` of a writer's core, never a system head.
+
+Each flush stamps the head you vouch for into your own oplog, and peers read those stamps out of the writers they wake up on, so trust travels with the log.
+
+#### `isTrusted(key, reference)`
+
+Return whether `key` is a writer you trust, judged against the `reference` view.
+
+Positive answers are cached until an undo rewinds the view, and the default is `true` unless you supply `mostRecentTrusted`, in which case it is `false`.
+
+#### `mostRecentTrusted(target, reference)`
+
+Return the oplog head you most recently vouched for, given the `target` view being considered.
+
+Called at flush time to stamp your own oplog (with your view as `target` and a `null` reference), and again per candidate during discovery.
+
+#### `fastForward.boot`
+
+```js
+{
+  head: { key, length },       // oplog head to boot from
+  legacy: { key, length },     // pre-2.0 pointer, see below
+  bootCondition (target, reference) {} // optional gate on the view we would land on
+}
+```
+
+Pass one of `head` or `legacy`, not both.
+
+Without `bootCondition` this is a single attempt that gives up if the head cannot be read; with one it parks until the condition is satisfied, which today means indefinitely if it never is.
+
+`legacy` is for records written before boot heads were oplog heads: the key is a _system_ head and a `0` length is resolved from the core. It boots ungated - `bootCondition` does not apply - and will be removed, so don't reach for it. A bare `{ key, length }` in place of the whole struct means the same thing, older still.
+
+#### `fastForward.conservative`
+
+Defaults to `true`: only fast-forward onto a head a connected peer can serve whole.
+
+The check covers the oplog head only, not the system and view cores the fast-forward then reads.
+
+#### `await db.moveTo(head)`
+
+Fast-forward onto an oplog head, ignoring the usual distance and conservative checks. Resolves `{ to, from }`.
+
 ### Encryption
 
 Pass an `encryptionKey` to encrypt all writer cores and the view at rest.

package/encoding/spec/autobee/index.js

@@ -856,15 +856,28 @@ const encoding29_7 = c.array(c.frame(encoding28))
 // @autobee/oplog-message-v3
 const encoding29 = {
   preencode(state, m) {
+    let flags =
+      (m.batch ? 1 : 0) |
+      (m.views ? 2 : 0) |
+      (m.optimistic ? 8 : 0) |
+      (m.value ? 16 : 0) |
+      (m.witness ? 32 : 0) |
+      (m.attestations ? 64 : 0) |
+      (m.trusted ? 128 : 0)
+    if (m.views) {
+      flags |= m.views.view ? 4 : 0
+    }
+
     c.uint.preencode(state, m.timestamp)
     encoding29_1.preencode(state, m.links)
-    state.end++ // max flag is 64 so always one byte
+    c.uint.preencode(state, flags)
 
     if (m.batch) encoding24.preencode(state, m.batch)
     if (m.views) encoding25_inline.preencode(state, m.views)
     if (m.value) c.buffer.preencode(state, m.value)
     if (m.witness) encoding29_6.preencode(state, m.witness)
     if (m.attestations) encoding29_7.preencode(state, m.attestations)
+    if (m.trusted) encoding29_8.preencode(state, m.trusted)
   },
   encode(state, m) {
     let flags =
@@ -873,7 +886,8 @@ const encoding29 = {
       (m.optimistic ? 8 : 0) |
       (m.value ? 16 : 0) |
       (m.witness ? 32 : 0) |
-      (m.attestations ? 64 : 0)
+      (m.attestations ? 64 : 0) |
+      (m.trusted ? 128 : 0)
     if (m.views) {
       flags |= m.views.view ? 4 : 0
     }
@@ -887,6 +901,7 @@ const encoding29 = {
     if (m.value) c.buffer.encode(state, m.value)
     if (m.witness) encoding29_6.encode(state, m.witness)
     if (m.attestations) encoding29_7.encode(state, m.attestations)
+    if (m.trusted) encoding29_8.encode(state, m.trusted)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -903,7 +918,8 @@ const encoding29 = {
       optimistic: (flags & 8) !== 0,
       value: (flags & 16) !== 0 ? c.buffer.decode(state) : null,
       witness: (flags & 32) !== 0 ? encoding29_6.decode(state) : null,
-      attestations: (flags & 64) !== 0 ? encoding29_7.decode(state) : null
+      attestations: (flags & 64) !== 0 ? encoding29_7.decode(state) : null,
+      trusted: (flags & 128) !== 0 ? encoding29_8.decode(state) : null
     }
   }
 }
@@ -1007,12 +1023,39 @@ const encoding31 = {
   }
 }
 
+// @autobee/trusted-head
+const encoding32 = {
+  preencode(state, m) {
+    c.fixed32.preencode(state, m.key)
+    c.uint.preencode(state, m.length)
+    c.uint.preencode(state, m.flushes)
+  },
+  encode(state, m) {
+    c.fixed32.encode(state, m.key)
+    c.uint.encode(state, m.length)
+    c.uint.encode(state, m.flushes)
+  },
+  decode(state) {
+    const r0 = c.fixed32.decode(state)
+    const r1 = c.uint.decode(state)
+    const r2 = c.uint.decode(state)
+
+    return {
+      key: r0,
+      length: r1,
+      flushes: r2
+    }
+  }
+}
+
 // @autobee/system-info-v3.heads, deferred due to recusive use
 const encoding18_3 = c.array(encoding22)
 // @autobee/system-info-v3.indexers, deferred due to recusive use
 const encoding18_4 = encoding18_3
 // @autobee/oplog-message-v3.links, deferred due to recusive use
 const encoding29_1 = encoding18_3
+// @autobee/oplog-message-v3.trusted, deferred due to recusive use
+const encoding29_8 = c.array(c.frame(encoding32))
 
 function setVersion(v) {
   version = v
@@ -1101,6 +1144,8 @@ function getEncoding(name) {
       return encoding30
     case '@autobee/manifest-data':
       return encoding31
+    case '@autobee/trusted-head':
+      return encoding32
     default:
       throw new Error('Encoder not found ' + name)
   }

package/encoding/spec/autobee/schema.json

@@ -750,6 +750,12 @@
           "array": true,
           "type": "@autobee/attestation",
           "version": 1
+        },
+        {
+          "name": "trusted",
+          "array": true,
+          "type": "@autobee/trusted-head",
+          "version": 1
         }
       ]
     },
@@ -804,6 +810,32 @@
           "version": 1
         }
       ]
+    },
+    {
+      "name": "trusted-head",
+      "namespace": "autobee",
+      "compact": false,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "key",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "flushes",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
     }
   ],
   "namespaces": [

package/index.js

@@ -19,11 +19,12 @@ const System = require('./lib/system.js')
 const ApplyCalls = require('./lib/apply-calls.js')
 const topo = require('./lib/topo.js')
 const { ActiveWriters } = require('./lib/writers.js')
+const TrustedPeers = require('./lib/trusted.js')
+const ApplyView = require('./lib/apply-view.js')
 const UpdateChanges = require('./lib/updates.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const INTERRUPT = new Error('Apply interrupted')
-const MIN_FF_GAP = 32
 
 module.exports = class Autobee extends ReadyResource {
   constructor(store, key = null, handlers = {}) {
@@ -56,7 +57,7 @@ module.exports = class Autobee extends ReadyResource {
     this.discoveryKey = null
     this.id = null
     this.bootstrap = null
-    this.handlers = handlers
+    this._handlers = handlers
     this.stats = { undos: 0, fastForwards: 0, drains: 0, applies: 0, appends: 0 }
 
     this.system = new System(this.store.namespace('system'), this.name, {
@@ -66,7 +67,7 @@ module.exports = class Autobee extends ReadyResource {
     this.system.auto = this
 
     this.bee = bee.snapshot()
-    this.view = handlers.open ? handlers.open(this.bee, this) : this.bee
+    this.view = ApplyView.open(this.bee, this)
     this.optimistic = handlers.optimistic !== false // TODO: should default to false instead
 
     this.name = name // for debugging
@@ -79,18 +80,20 @@ module.exports = class Autobee extends ReadyResource {
 
     const fastForward = handlers.fastForward || {}
 
-    // system head to boot from: migrates or fast-forwards depending on its version
+    // oplog head to boot from: migrates or fast-forwards depending on its version
     this.bootFrom = fastForward.boot || null
 
     // conservative (default on): only fast-forward onto a head someone can serve whole
     this._conservativeFF = fastForward.conservative !== false
 
+    this.trusted = new TrustedPeers(handlers)
+
     this.ff = null
     this.fastForwarding = null
     this.fastForwardTo = null
 
     this._workingBee = bee
-    this._workingView = handlers.open ? handlers.open(this._workingBee, this) : this._workingBee
+    this._workingView = new ApplyView(this._workingBee, this)
 
     this._localSystemStart = 0
     this._localSystemLength = 0
@@ -100,6 +103,7 @@ module.exports = class Autobee extends ReadyResource {
 
     this._appending = []
     this._draining = null
+    this._bootWait = null
 
     this.legacyViews = handlers.legacyViews || []
 
@@ -107,7 +111,6 @@ module.exports = class Autobee extends ReadyResource {
     this._bootingState = null
     this._bootingAll = null
 
-    this._handlers = handlers
     this._now = handlers.now || Date.now // overridable for clock-drift tests
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
@@ -175,6 +178,11 @@ module.exports = class Autobee extends ReadyResource {
     this.bumpSoon()
   }
 
+  _requestWakeup() {
+    const session = this._wakeup._session
+    if (session) session.broadcastLookup()
+  }
+
   _registerWakeup() {
     if (!this.wakeupCapability) return
     this._wakeup.recouple()
@@ -219,6 +227,7 @@ module.exports = class Autobee extends ReadyResource {
 
   async _close() {
     this._interrupting = true
+    if (this._bootWait !== null) this._bootWait.resolve()
     if (this._notifyHandler) this._notifyHandler.destroy()
     if (this._draining) {
       // drain may be waiting on replicated blocks
@@ -231,14 +240,14 @@ module.exports = class Autobee extends ReadyResource {
       await this._bootingAll
     } catch {}
 
-    if (this._handlers.close) await this._handlers.close(this.view)
+    await ApplyView.close(this.view, this)
 
     if (this.writers) await this.writers.close()
     await this.local.close()
     await this.system.close()
     await this._wakeup.close()
     if (this.bootstrap && !this.bootstrap.closed) await this.bootstrap.close()
-    await this._workingBee.close()
+    await this._workingView.close()
     await this.bee.close()
     await this.store.close()
   }
@@ -287,6 +296,14 @@ module.exports = class Autobee extends ReadyResource {
     this._wakeup.hint(wakeup)
   }
 
+  // the view an oplog head points at, opened and closed through the handlers
+  openViewAt(oplog) {
+    const v = oplog.op.views.view
+    if (!v) return null
+
+    return new ApplyView(this.bee.checkout({ key: v.key, length: v.start + v.length }), this)
+  }
+
   openCore(key) {
     const encryption = this.encryptionKey ? new WriterEncryption(this) : null
     return this.store.get({ key, encryption })
@@ -312,7 +329,7 @@ module.exports = class Autobee extends ReadyResource {
     }
 
     if (this.bootFrom) {
-      this.bootFrom = (await this.bootFrom) || null
+      this.bootFrom = getBootOption(await this.bootFrom)
     }
   }
 
@@ -400,8 +417,8 @@ module.exports = class Autobee extends ReadyResource {
 
     // @todo migration
     if (result.migration) {
-      if (this.handlers.migrate) {
-        view = (await this.handlers.migrate(result.migration.views)) || EMPTY_HEAD
+      if (this._handlers.migrate) {
+        view = (await this._handlers.migrate(result.migration.views)) || EMPTY_HEAD
         this._catchupMigratedNodes = result.migration.catchup
         await this._storeMigratedView(view)
       }
@@ -443,6 +460,8 @@ module.exports = class Autobee extends ReadyResource {
   async _bump() {
     if (!(await this._bootReady())) return
 
+    if (this._bootWait !== null) this._bootWait.resolve()
+
     this.bumping++
 
     if (!this._draining) {
@@ -502,8 +521,12 @@ module.exports = class Autobee extends ReadyResource {
     this.stats.drains++
 
     if (this.bootFrom) {
-      await this._initFromHead(this.bootFrom)
+      const { head = null, legacy = null, bootCondition = null } = this.bootFrom
+
       this.bootFrom = null
+
+      if (legacy) await this._bootFromSystem(legacy)
+      else if (head) await this._bootFromHead(head, bootCondition)
     }
 
     // tracked across drain iteration
@@ -607,49 +630,86 @@ module.exports = class Autobee extends ReadyResource {
     }
 
     // a scheduled fast-forward is applied by the drain before we look again
-    if (this.fastForwardTo !== null) return
-    if (this._interrupting || this.closing) return
+    if (this.fastForwardTo !== null || this.fastForwarding !== null) return
+    if (this._interrupting || this.closing || this.bootFrom) return
 
     try {
-      await this.fastForwardFromHeads(hints, { wait: false, timeout: FastForward.DEFAULT_TIMEOUT })
+      const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
+
+      const ff = await FastForward.fromHeads(this, heads, {
+        timeout: FastForward.DEFAULT_TIMEOUT
+      })
+
+      if (ff !== null) await this._runFastForward(ff)
     } catch (err) {
       safetyCatch(err)
-      return false
     }
+  }
 
-    return true
+  async _readCandidateHeads(hints, timeout) {
+    const promises = []
+
+    for (const [hex, length] of hints) {
+      if (length === 0) continue
+      const key = b4a.from(hex, 'hex')
+      promises.push(this._getOplog(key, length, timeout ? { timeout } : null))
+    }
+
+    const ops = await Promise.all(promises)
+    const heads = []
+
+    for (const res of ops) {
+      if (res === null) continue
+
+      // the head we were woken on is a candidate in its own right
+      heads.push({ key: res.key, length: res.length })
+
+      if (!res.op.trusted) continue
+
+      for (const trusted of res.op.trusted) {
+        const head = this.trusted.read(trusted)
+        if (head !== null) heads.push(head)
+      }
+    }
+
+    return heads
   }
 
   async _getOplog(key, length, opts = null) {
     const core = this.openCore(key)
 
     try {
-      await core.ready()
+      return await this.readOplog(core, length, opts)
+    } finally {
+      await core.close()
+    }
+  }
+
+  // reads on a core the caller owns, so a walk does not churn a session per step
+  async readOplog(core, length, opts = null) {
+    await core.ready()
 
-      const target = length >= 0 ? length : core.length
-      if (target === 0) return null
+    const target = length >= 0 ? length : core.length
+    if (target === 0) return null
 
-      // conservative: only proceed when a connected peer can serve the head whole
-      if (opts && opts.conservative && core.remoteContiguousLength < target) return null
+    // conservative: only proceed when a connected peer can serve the head whole
+    if (opts && opts.conservative && core.remoteContiguousLength < target) return null
 
-      const buf = await core.get(target - 1, opts)
-      if (buf === null) return null
+    const buf = await core.get(target - 1, opts)
+    if (buf === null) return null
 
-      let op = encoding.decodeOplog(buf)
+    let op = encoding.decodeOplog(buf)
 
-      // legacy nodes always inflate, but only indexers carry views - callers
-      // that need system info must check op.views
-      if (op.version < 3) {
-        op = await this._inflateLegacyOplog(buf, core, target - 1, opts)
-      }
+    // legacy nodes always inflate, but only indexers carry views - callers
+    // that need system info must check op.views
+    if (op.version < 3) {
+      op = await this._inflateLegacyOplog(buf, core, target - 1, opts)
+    }
 
-      return {
-        key,
-        length: target,
-        op
-      }
-    } finally {
-      await core.close()
+    return {
+      key: core.key,
+      length: target,
+      op
     }
   }
 
@@ -939,6 +999,7 @@ module.exports = class Autobee extends ReadyResource {
 
     if (t.undo) {
       this.stats.undos++
+      this.trusted.clear()
       t.view = await this.system.undo(t.undo)
     }
 
@@ -989,7 +1050,7 @@ module.exports = class Autobee extends ReadyResource {
     if (this._hasApply && (await this.system.canApply(batch[0].key, optimistic))) {
       this.stats.applies++
       this._host.applying = batch
-      await this._handlers.apply(userBatch, this._workingView, this._host)
+      await this._workingView.apply(userBatch)
       this._host.applying = null
     }
 
@@ -1135,115 +1196,108 @@ module.exports = class Autobee extends ReadyResource {
     this._localViewLength = 0
   }
 
-  async fastForwardFromHeads(hints, { wait = true, timeout = 0 } = {}) {
-    if (!this._handlers.onwakeup) return false
-    if (this._interrupting) return false
-    if (!hints.size || this.fastForwarding || this.fastForwardTo || this.bootFrom) {
-      return false
-    }
+  async moveTo(head) {
+    if (!this.opened) await this.ready()
+    if (this.closing) throw new Error('Autobee closed')
 
-    const promises = []
-    const heads = []
-    for (const [hex, length] of hints) {
-      if (length === 0) continue
-      const key = b4a.from(hex, 'hex')
-      heads.push({ key, length })
-      promises.push(this._getOplog(key, length, timeout ? { timeout } : null))
-    }
+    const ff = await FastForward.fromHead(this, head, null, { force: true })
+    if (ff === null) return null
 
-    const ops = await Promise.all(promises)
-    if (this.fastForwarding || this.fastForwardTo) return false
+    if (!(await this._runFastForward(ff))) return null
 
-    let best = null
-    let bestFlushes = -1
+    return this.ff.promise
+  }
 
-    for (let i = 0; i < ops.length; i++) {
-      const res = ops[i]
-      if (res === null) continue
+  // legacy: ungated boot straight onto a system head
+  async _bootFromSystem(system) {
+    try {
+      const ff = new FastForward(this, system, null, { timeout: FastForward.DEFAULT_TIMEOUT })
+      return await this._runFastForward(ff)
+    } catch (err) {
+      safetyCatch(err)
+      return false
+    }
+  }
 
-      if (res.op.views && res.op.views.flushes > bestFlushes) {
-        bestFlushes = res.op.views.flushes
-        best = res
+  // the boot head seeds the view every trust decision here is made against,
+  // since our own view is still empty
+  async _bootFromHead(head, bootCondition) {
+    // just a head: one attempt, we do not wait around if it cannot be read
+    if (bootCondition === null) {
+      try {
+        const ff = await FastForward.fromHead(this, head, null, {
+          force: true,
+          timeout: FastForward.DEFAULT_TIMEOUT
+        })
+
+        return ff !== null && (await this._runFastForward(ff))
+      } catch (err) {
+        safetyCatch(err)
+        return false
       }
     }
 
-    if (best === null || bestFlushes - this.system.flushes < MIN_FF_GAP) return false
-
-    const head = { key: best.key, length: best.length }
-
-    const v = best.op.views.view
-    const view = this.bee.checkout({ key: v.key, length: v.start + v.length })
+    // a condition needs the view at head to check against, so park until we
+    // can read it and something acceptable turns up
+    let opened = null
 
-    let trusted = null
     try {
-      trusted = await this._handlers.onwakeup(head, view, this)
-      if (!trusted || this.fastForwarding || this.fastForwardTo) return false
-    } finally {
-      view.close()
-    }
+      while (!this._interrupting) {
+        this._bootWait = rrp()
 
-    return this._fastForwardFromHead(best, trusted, { wait, timeout })
-  }
+        try {
+          if (opened === null) opened = await this._bootReference(head)
 
-  async _fastForwardFromHead(head, trusted, { force = false, wait = true, timeout = 0 } = {}) {
-    const conservative = !force && this._conservativeFF
-    const timeoutOpts = timeout ? { timeout } : null
-
-    const oplog = await this._getOplog(head.key, head.length, {
-      conservative,
-      ...timeoutOpts
-    })
-    if (!oplog) return null
-
-    const verified = trusted
-      ? await this._getOplog(trusted.key, trusted.length, timeoutOpts)
-      : oplog
-    if (!verified) return null
-
-    // legacy nodes from non-indexers have no system info to fast-forward from
-    if (!oplog.op.views || !verified.op.views) return null
+          if (opened !== null && (await this._bootAttempt(head, bootCondition, opened.view))) {
+            return true
+          }
+        } catch (err) {
+          safetyCatch(err)
+        }
 
-    if (!force && verified.op.views.flushes - this.system.flushes < MIN_FF_GAP) {
-      return null
-    }
+        if (this._interrupting) break
 
-    const tip = {
-      system: batchToHead(oplog.op.views.system),
-      verified: {
-        op: trusted || head,
-        flushes: verified.op.views.flushes
+        await this._bootWait.promise
       }
+    } finally {
+      this._bootWait = null
+      if (opened !== null) await opened.close()
     }
 
-    const moved = await this._moveTo(batchToHead(verified.op.views.system), tip)
-
-    if (moved && wait) return this.ff.promise
-
-    return null
+    return false
   }
 
-  // same as moveTo except we don't return the final promise
-  _initFromHead(head, tip) {
-    if (!head.length) {
-      // legacy fastForward boot
-      return this._runFastForward(new FastForward(this, head, tip))
-    }
+  async _bootAttempt(head, bootCondition, reference) {
+    // peek, so the hints are still there for the drain once we have booted
+    const candidates = await this._readCandidateHeads(
+      this._wakeup.hints,
+      FastForward.DEFAULT_TIMEOUT
+    )
 
-    return this._fastForwardFromHead(head, null, { force: true, wait: false })
-  }
+    const ff = await FastForward.fromHeads(this, [head, ...candidates], {
+      force: true,
+      timeout: FastForward.DEFAULT_TIMEOUT,
+      condition: bootCondition,
+      reference
+    })
 
-  // head is a system head; fast-forwards onto it, migrating in place if it's a legacy version
-  async _moveTo(systemHead, tip) {
-    if (this.fastForwardTo !== null || this.fastForwarding !== null) return false
+    return ff !== null && (await this._runFastForward(ff))
+  }
 
-    if (await this._runFastForward(new FastForward(this, systemHead, tip))) {
-      return true
-    }
+  async _bootReference(head) {
+    const oplog = await FastForward.flushHead(this, head, {
+      timeout: FastForward.DEFAULT_TIMEOUT
+    })
 
-    return false
+    return oplog === null ? null : this.openViewAt(oplog)
   }
 
   async _runFastForward(ff) {
+    if (this.fastForwardTo !== null || this.fastForwarding !== null) {
+      await ff.close()
+      return false
+    }
+
     this.fastForwarding = ff
 
     const result = await ff.run()
@@ -1281,7 +1335,7 @@ module.exports = class Autobee extends ReadyResource {
 
     // migrate is set when fast-forwarding from a legacy head
     if (migrate) {
-      const view = (await this.handlers.migrate(migrate)) || EMPTY_HEAD
+      const view = (await this._handlers.migrate(migrate)) || EMPTY_HEAD
       await this._storeMigratedView(view)
       this.bee.move(view)
       this._workingBee.move(view)
@@ -1293,6 +1347,9 @@ module.exports = class Autobee extends ReadyResource {
     this.fastForwardTo = null
     await this.writers.refresh()
 
+    // we moved, so ask our peers to tell us their heads again
+    this._requestWakeup()
+
     await this._storeBoot()
 
     await this._update(changes)
@@ -1333,6 +1390,17 @@ function isObject(o) {
   return typeof o === 'object' && o && !b4a.isBuffer(o)
 }
 
+function getBootOption(boot) {
+  if (!boot) return null
+
+  // oldest style, supported for now but will go away: a bare key is legacy
+  if (boot.key) return { legacy: boot, head: null, bootCondition: null }
+
+  asserts.assert(!(boot.head && boot.legacy), 'Boot from either a head or a legacy pointer')
+
+  return boot
+}
+
 function noop() {}
 
 function createAnchorCore(store, prologue, manifestData) {
@@ -1361,10 +1429,3 @@ function crashSoon(err) {
   })
   throw err
 }
-
-function batchToHead(b) {
-  return {
-    key: b.key,
-    length: b.start + b.length
-  }
-}

package/lib/apply-view.js

@@ -0,0 +1,25 @@
+module.exports = class ApplyView {
+  constructor(bee, auto) {
+    this.bee = bee
+    this.auto = auto
+    this.view = ApplyView.open(bee, auto)
+  }
+
+  static open(bee, auto) {
+    return auto._handlers.open ? auto._handlers.open(bee, auto) : bee
+  }
+
+  static close(view, auto) {
+    if (!auto._handlers.close) return Promise.resolve()
+    return auto._handlers.close(view)
+  }
+
+  apply(userBatch) {
+    return this.auto._handlers.apply(userBatch, this.view, this.auto._host)
+  }
+
+  async close() {
+    await ApplyView.close(this.view, this.auto)
+    await this.bee.close()
+  }
+}

package/lib/fast-forward.js

@@ -19,6 +19,11 @@ const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
 
 const EMPTY = b4a.alloc(0)
 const DEFAULT_OP_TIMEOUT = 5_000
+const MIN_FF_GAP = 32
+
+// views are only stamped on the last node of a flush, so a head may sit above
+// the most recent one - how far back we look for it
+const MAX_FLUSH_SEARCH = 64
 
 module.exports = class FastForward {
   constructor(auto, head, tip, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
@@ -46,6 +51,162 @@ module.exports = class FastForward {
 
   static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
 
+  static async fromHead(
+    auto,
+    head,
+    trusted,
+    { force = false, timeout = 0, condition = null, reference = null } = {}
+  ) {
+    const conservative = !force && auto._conservativeFF
+    const timeoutOpts = timeout ? { timeout } : null
+
+    const oplog = await FastForward.flushHead(auto, head, {
+      conservative,
+      ...timeoutOpts
+    })
+    if (!oplog) return null
+
+    const verified = trusted ? await FastForward.flushHead(auto, trusted, timeoutOpts) : oplog
+    if (!verified) return null
+
+    // legacy nodes from non-indexers have no system info to fast-forward from
+    if (!oplog.op.views || !verified.op.views) return null
+
+    if (!force && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
+      return null
+    }
+
+    if (condition !== null && !(await FastForward.accepts(auto, verified, condition, reference))) {
+      return null
+    }
+
+    const tip = {
+      system: batchToHead(oplog.op.views.system),
+      verified: {
+        op: { key: verified.key, length: verified.length },
+        flushes: verified.op.views.flushes
+      }
+    }
+
+    return new FastForward(auto, batchToHead(verified.op.views.system), tip)
+  }
+
+  static async fromHeads(
+    auto,
+    heads,
+    { force = false, timeout = 0, condition = null, reference = null } = {}
+  ) {
+    if (reference === null) reference = auto._workingView.view
+
+    const promises = []
+
+    for (const head of heads) {
+      if (head.length === 0) continue
+      promises.push(FastForward.flushHead(auto, head, timeout ? { timeout } : null))
+    }
+
+    const ops = await Promise.all(promises)
+    if (auto.fastForwarding || auto.fastForwardTo) return null
+
+    const trust = await Promise.all(
+      ops.map((res) => {
+        return res === null ? false : auto.trusted.isTrusted(res.key, reference)
+      })
+    )
+    if (auto.fastForwarding || auto.fastForwardTo) return null
+
+    const candidates = []
+
+    let bestTrusted = null
+    let bestTrustedFlushes = -1
+
+    for (let i = 0; i < ops.length; i++) {
+      const res = ops[i]
+      if (res === null || !res.op.views) continue
+      if (!force && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
+
+      if (!trust[i]) {
+        candidates.push(res)
+        continue
+      }
+
+      if (res.op.views.flushes > bestTrustedFlushes) {
+        bestTrustedFlushes = res.op.views.flushes
+        bestTrusted = res
+      }
+    }
+
+    if (bestTrusted !== null) {
+      return FastForward.fromHead(auto, bestTrusted, null, {
+        force,
+        timeout,
+        condition,
+        reference
+      })
+    }
+
+    candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
+
+    for (const res of candidates) {
+      const trusted = await FastForward.mostRecentTrusted(auto, res, reference)
+      if (trusted === null) continue
+      if (auto.fastForwarding || auto.fastForwardTo) return null
+
+      const ff = await FastForward.fromHead(auto, res, trusted, {
+        force,
+        timeout,
+        condition,
+        reference
+      })
+      if (ff !== null) return ff
+    }
+
+    return null
+  }
+
+  static async flushHead(auto, head, opts) {
+    const core = auto.openCore(head.key)
+
+    try {
+      let length = head.length
+
+      for (let i = 0; i < MAX_FLUSH_SEARCH; i++) {
+        const oplog = await auto.readOplog(core, length, opts)
+        if (oplog === null) return null
+        if (oplog.op.views) return oplog
+
+        length = oplog.length - 1
+        if (length <= 0) return null
+      }
+
+      return null
+    } finally {
+      await core.close()
+    }
+  }
+
+  static async accepts(auto, oplog, condition, reference) {
+    const opened = auto.openViewAt(oplog)
+    if (opened === null) return false
+
+    try {
+      return !!(await condition(opened.view, reference))
+    } finally {
+      await opened.close()
+    }
+  }
+
+  static async mostRecentTrusted(auto, head, reference) {
+    const opened = auto.openViewAt(head)
+    if (opened === null) return null
+
+    try {
+      return (await auto.trusted.mostRecentTrusted(opened.view, reference)) || null
+    } finally {
+      await opened.close()
+    }
+  }
+
   async run() {
     try {
       if (!this.running) this.running = this._run()
@@ -228,6 +389,13 @@ module.exports = class FastForward {
   }
 }
 
+function batchToHead(b) {
+  return {
+    key: b.key,
+    length: b.start + b.length
+  }
+}
+
 function decodeLegacySystemInfo(buffer) {
   const state = { start: 0, end: buffer.length, buffer }
   const version = c.uint.decode(state)

package/lib/topo.js

@@ -308,7 +308,7 @@ function getLocalBatch(undo, writer, length, weight, index) {
 
   if (length > writer.core.length) {
     if (!writer.pending) {
-      throw new Error(`No local nodes: looking for ${length}, found ${writer.core.length}`)
+      throw new Error(`No local nodes: looking for ${length}, found ${writer.core.length}@${writer.core.id}`)
     }
 
     let i = 0

package/lib/trusted.js

@@ -0,0 +1,44 @@
+const b4a = require('b4a')
+
+class TrustedPeers {
+  constructor(hooks) {
+    this.trusted = new Map()
+    this.pending = new Map()
+    this.hooks = hooks
+  }
+
+  async isTrusted(key, reference) {
+    const hex = b4a.toString(key, 'hex')
+    if (this.trusted.get(hex)) return true
+
+    let promise = this.pending.get(hex)
+    if (promise) return promise
+
+    promise = this.hooks.isTrusted
+      ? this.hooks.isTrusted(key, reference)
+      : Promise.resolve(!this.hooks.mostRecentTrusted)
+    this.pending.set(hex, promise)
+
+    const trusted = await promise
+    if (trusted) this.trusted.set(hex, trusted)
+
+    if (this.pending.get(hex) === promise) this.pending.delete(hex)
+    return trusted
+  }
+
+  read(trusted) {
+    return trusted || null
+  }
+
+  mostRecentTrusted(target, reference) {
+    if (!this.hooks.mostRecentTrusted) return Promise.resolve(null)
+    return this.hooks.mostRecentTrusted(target, reference)
+  }
+
+  clear() {
+    this.pending.clear()
+    this.trusted.clear()
+  }
+}
+
+module.exports = TrustedPeers

package/lib/writers.js

@@ -64,6 +64,9 @@ class ActiveWriters {
       w = await this.add(key)
     }
 
+    // prefetch the oplog head for the fast-forward that runs right after us
+    if (w && length > 0) w.core.get(length - 1).catch(noop)
+
     // TODO: signal that we've seen length on w for gc
 
     this.bump()
@@ -95,16 +98,18 @@ class ActiveWriters {
     return this.localWriter.clear()
   }
 
-  flushLocal(views) {
+  async flushLocal(views) {
     const w = this.localWriter
 
     // only consume the queue when a flush will actually write
-    if (w.pending === null || w.processed === 0) return Promise.resolve()
+    if (w.pending === null || w.processed === 0) return
 
     const queued = this.attestations
     this.attestations = []
 
-    return w.flush(views, queued)
+    const trusted = await this.auto.trusted.mostRecentTrusted(this.auto._workingView.view, null)
+
+    return w.flush(views, queued, trusted)
   }
 
   async refresh() {
@@ -623,7 +628,8 @@ class Writer {
       batch,
       views: null,
       optimistic: optimistic && !this.writers.writable,
-      value
+      value,
+      trusted: null
     }
 
     const node = createNode(this.core, this.appendLength++, -1, oplog)
@@ -646,12 +652,16 @@ class Writer {
     this.processed = 0
   }
 
-  flush(views, queued) {
+  flush(views, queued, trusted) {
     if (this.pending === null || this.processed === 0) return Promise.resolve()
 
     const head = this.pending[this.processed - 1]
     head.views = views
 
+    head.trusted = trusted
+      ? [{ key: trusted.key, length: trusted.length, flushes: views.flushes }]
+      : null
+
     if (queued && queued.length) {
       const attestations = []
       for (const { key, weight } of queued) {
@@ -695,7 +705,8 @@ function createNode(core, length, weight, oplog) {
     batch: oplog.batch,
     views: oplog.views,
     optimistic: oplog.optimistic,
-    value: oplog.value
+    value: oplog.value,
+    trusted: oplog.trusted || []
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.15",
+  "version": "2.0.0-rc.16",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",