autobee 2.7.0 → 2.7.2

package/README.md

@@ -256,13 +256,17 @@ Return the oplog head you most recently vouched for, given the `target` view bei
 
 Called at flush time to stamp your own oplog (with your view as `target` and a `null` reference), and again per candidate during discovery.
 
+During discovery `target` is a view opened for the candidate, and reads through it carry the fast-forward's timeout. Anything else the hook awaits is unbounded, so if it reaches beyond that view, put a timeout on it: the candidate waits on the hook, and so does closing it.
+
 #### `warmup(view)`
 
 Prepare a candidate `view` before the fast-forward onto it is applied.
 
 Called as soon as the candidate's view head is known, and runs alongside the block downloads the fast-forward is already doing rather than after them. Throw or reject to reject the candidate: the fast-forward is abandoned and the usual apply path catches up instead.
 
-`view` is opened and closed through the `open` and `close` handlers, so don't build or close a db of your own.
+`view` is opened and closed through the `open` and `close` handlers, so don't build or close a db of your own. Reads through it carry the fast-forward's timeout, so a warmup that needs blocks nobody serves fails the candidate instead of hanging.
+
+Only reads through `view` are bounded. Anything else the handler awaits, such as other cores or your own network calls, is up to you to time out: the fast-forward waits on the handler, and an abandoned candidate cannot finish closing until it returns.
 
 #### `fastForward.boot`
 
@@ -281,9 +285,9 @@ Pass one of `head` or `legacy`, not both.
 
 #### `fastForward.conservative`
 
-Defaults to `true`: only fast-forward onto a head a connected peer can serve whole.
+Defaults to `true`: only fast-forward once the system and view lengths the head stamps are held whole, either locally or by a connected peer.
 
-The check covers the oplog head only, not the system and view cores the fast-forward then reads.
+The check runs at the end of the fast-forward, after the system has booted and the view head has been fetched, so a sparse view nobody can serve is skipped instead of landed on.
 
 #### `await db.moveTo(head, [options])`
 

package/index.js

@@ -98,8 +98,8 @@ module.exports = class Autobee extends ReadyResource {
     // oplog head to boot from: migrates or fast-forwards depending on its version
     this.bootFrom = (fastForward && fastForward.boot) || null
 
-    // conservative (default on): only fast-forward onto a head someone can serve whole
-    this._conservativeFF = !fastForward || !!fastForward.conservative
+    // conservative (default on): only fast-forward onto a system and view held whole
+    this._conservativeFF = fastForward === null || fastForward.conservative !== false
 
     this.trusted = new TrustedPeers(this, handlers)
 
@@ -437,15 +437,17 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   // the view an oplog head points at, opened and closed through the handlers
-  openViewAt(oplog) {
+  openViewAt(oplog, { timeout } = {}) {
     const v = oplog.op.views.view
     if (!v) return null
 
-    return this.openView({ key: v.key, length: v.start + v.length })
+    return this.openView({ key: v.key, length: v.start + v.length }, { timeout })
   }
 
-  openView(head) {
-    return new ApplyView(this.bee.checkout({ key: head.key, length: head.length }), this)
+  // timeout bounds every hypercore read done through the view, defaults to the main bee's
+  openView(head, { timeout } = {}) {
+    const bee = this.bee.checkout({ key: head.key, length: head.length, timeout })
+    return new ApplyView(bee, this)
   }
 
   openCore(key) {
@@ -826,7 +828,7 @@ module.exports = class Autobee extends ReadyResource {
 
     for (const { key, length } of hints) {
       if (length === 0) continue
-      promises.push(this._resolveOplogHint(key, length, timeout ? { timeout } : null))
+      promises.push(this._resolveOplogHint(key, length, { timeout }))
     }
 
     const ops = await Promise.all(promises)
@@ -865,7 +867,7 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _getOplog(key, length, opts = null) {
+  async _getOplog(key, length, opts) {
     const core = this.openCore(key)
 
     try {
@@ -876,17 +878,12 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   // reads on a core the caller owns, so a walk does not churn a session per step
-  async readOplog(core, length, opts = null) {
+  async readOplog(core, length, { timeout = 0 } = {}) {
     await core.ready()
 
-    const { conservative = false, timeout } = opts === null ? {} : opts
-
     const target = length >= 0 ? length : core.length
     if (target === 0) return null
 
-    // conservative: only proceed when a connected peer can serve the head whole
-    if (conservative && core.remoteContiguousLength < target) return null
-
     const buf = await core.get(target - 1, { timeout })
     if (buf === null) return null
 
@@ -1216,16 +1213,17 @@ module.exports = class Autobee extends ReadyResource {
     return t
   }
 
-  async applyBacklog(batches) {
-    const queue = batches.slice()
+  async _processBatch(batch) {
+    // a stack: the tip is pushed in reverse so it pops in order
+    const stack = [batch]
 
-    while (queue.length) {
-      const batch = queue.shift()
+    while (stack.length) {
+      const batch = stack.pop()
       const t = await this.prepareBatch(batch)
 
       if (t.view) {
         this._workingBee.move(t.view)
-        queue.unshift(...t.tip)
+        for (let i = t.tip.length - 1; i >= 0; i--) stack.push(t.tip[i])
         continue
       }
 
@@ -1235,15 +1233,12 @@ module.exports = class Autobee extends ReadyResource {
 
       await this._applyBatch(batch, batch[0].optimistic)
 
-      this.writers.triggers.trigger(
-        b4a.toString(batch[0].key, 'hex'),
-        batch[batch.length - 1].length
-      )
-    }
-  }
+      // trigger any upstream writer waiting for this batch early
+      const id = b4a.toString(batch[0].key, 'hex')
+      const length = batch[batch.length - 1].length
 
-  async _processBatch(batch) {
-    await this.applyBacklog([batch])
+      this.writers.triggers.trigger(id, length)
+    }
   }
 
   async _applyBatch(batch, optimistic) {

package/lib/fast-forward.js

@@ -10,9 +10,12 @@ const DEFAULT_OP_TIMEOUT = 5_000
 const MIN_FF_GAP = 32
 
 class FastForward {
-  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
+  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT, conservative = false } = {}) {
     this.auto = auto
 
+    // conservative: only land when the system and view we booted are held whole
+    this.conservative = conservative
+
     this.activeRequests = []
 
     // 'reboot' is a storage namespace: renaming it would derive a different
@@ -38,15 +41,11 @@ class FastForward {
 
   static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
     const conservative = !force && auto._conservativeFF
-    const timeoutOpts = timeout ? { timeout } : null
 
-    const oplog = await FastForward.flushHead(auto, head, {
-      conservative,
-      ...timeoutOpts
-    })
+    const oplog = await FastForward.flushHead(auto, head, { timeout })
     if (!oplog) return null
 
-    const verified = trusted ? await FastForward.flushHead(auto, trusted, timeoutOpts) : oplog
+    const verified = trusted ? await FastForward.flushHead(auto, trusted, { timeout }) : oplog
     if (!verified) return null
 
     // legacy nodes from non-indexers have no system info to fast-forward from
@@ -56,7 +55,10 @@ class FastForward {
       return null
     }
 
-    return new FastForward(auto, batchToHead(verified.op.views.system))
+    return new FastForward(auto, batchToHead(verified.op.views.system), {
+      timeout: timeout || DEFAULT_OP_TIMEOUT,
+      conservative
+    })
   }
 
   static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
@@ -66,7 +68,7 @@ class FastForward {
 
     for (const head of heads) {
       if (head.length === 0) continue
-      promises.push(FastForward.flushHead(auto, head, timeout ? { timeout } : null))
+      promises.push(FastForward.flushHead(auto, head, { timeout }))
     }
 
     const ops = await Promise.all(promises)
@@ -107,7 +109,7 @@ class FastForward {
     candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
 
     for (const res of candidates) {
-      const trusted = await FastForward.mostRecentTrusted(auto, res, reference)
+      const trusted = await FastForward.mostRecentTrusted(auto, res, reference, timeout)
       if (trusted === null) continue
       if (auto.fastForwarding || auto.fastForwardTo) return null
 
@@ -130,8 +132,8 @@ class FastForward {
     }
   }
 
-  static async mostRecentTrusted(auto, head, reference) {
-    const opened = auto.openViewAt(head)
+  static async mostRecentTrusted(auto, head, reference, timeout) {
+    const opened = auto.openViewAt(head, { timeout })
     if (opened === null) return null
 
     try {
@@ -178,8 +180,10 @@ class FastForward {
     promises.push(this.system.get(this.auto.local.key, { timeout: this.timeout }))
 
     // a legacy system without the designated view migrates onto an empty view
+    let view = null
+
     if (this.system.view.length) {
-      const view = this.auto.store.get({ key: this.system.view.key, active: true })
+      view = this.auto.store.get({ key: this.system.view.key, active: true })
       this.cores.push(view)
 
       promises.push(
@@ -197,6 +201,13 @@ class FastForward {
     await Promise.all(promises)
     if (this.destroyed) return null
 
+    // conservative: the lengths we land on must be held whole, either by us or
+    // by a connected peer, so the view does not stall on a block nobody has
+    if (this.conservative) {
+      if (!isContiguous(this.system.bee.core, this.head.length)) return null
+      if (view !== null && !isContiguous(view, this.system.view.length)) return null
+    }
+
     return {
       head: this.head,
       migrate: null
@@ -206,7 +217,7 @@ class FastForward {
   _startWarmup() {
     if (!this.auto._warmup || !this.system.view.length) return false
 
-    this.warmupView = this.auto.openView(this.system.view)
+    this.warmupView = this.auto.openView(this.system.view, { timeout: this.timeout })
 
     // ensure thenable
     this.warmup = Promise.resolve(this.auto._warmup(this.warmupView.view))
@@ -358,6 +369,10 @@ async function findNewestSystem(ff, info) {
   return { legacy, autobee }
 }
 
+function isContiguous(core, length) {
+  return core.contiguousLength >= length || core.remoteContiguousLength >= length
+}
+
 function batchToHead(b) {
   return {
     key: b.key,

package/lib/triggers.js

@@ -1,9 +1,14 @@
+// writers waiting on a node of another writer, keyed by that writer's id.
+// a writer waits on at most one entry at a time, tracked as writer.waiting
 module.exports = class Triggers {
   constructor() {
     this.waiting = new Map()
   }
 
   add(id, length, writer) {
+    // a writer only ever waits on one link: drop whatever it waited on before
+    if (writer.waiting !== null && writer.waiting.id !== id) this.remove(writer.waiting)
+
     let w = this.waiting.get(id)
 
     if (!w) {
@@ -24,6 +29,7 @@ module.exports = class Triggers {
   }
 
   remove(entry) {
+    // writer is nulled on removal, so a removed entry is a noop here
     if (!entry.writer) return
 
     const w = this.waiting.get(entry.id)
@@ -42,14 +48,18 @@ module.exports = class Triggers {
     const w = this.waiting.get(id)
     if (!w) return
 
+    // collect first: the walk swap-removes, so nothing may touch the array mid-loop
+    const bump = []
+
     for (let i = w.length - 1; i >= 0; i--) {
       const entry = w[i]
       if (entry.length > length) continue
 
-      const writer = entry.writer
+      bump.push(entry.writer)
       this._remove(w, entry)
-      writer.bump()
     }
+
+    for (const writer of bump) writer.bump()
   }
 
   _remove(w, entry) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.7.0",
+  "version": "2.7.2",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -27,7 +27,7 @@
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
     "compact-encoding": "^3.3.0",
-    "hyperbee2": "^2.10.3",
+    "hyperbee2": "^2.16.0",
     "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.20.1",