autobee 2.7.1 → 2.7.2

package/README.md

@@ -285,9 +285,9 @@ Pass one of `head` or `legacy`, not both.
 
 #### `fastForward.conservative`
 
-Defaults to `true`: only fast-forward onto a head a connected peer can serve whole.
+Defaults to `true`: only fast-forward once the system and view lengths the head stamps are held whole, either locally or by a connected peer.
 
-The check covers the oplog head only, not the system and view cores the fast-forward then reads.
+The check runs at the end of the fast-forward, after the system has booted and the view head has been fetched, so a sparse view nobody can serve is skipped instead of landed on.
 
 #### `await db.moveTo(head, [options])`
 

package/index.js

@@ -98,8 +98,8 @@ module.exports = class Autobee extends ReadyResource {
     // oplog head to boot from: migrates or fast-forwards depending on its version
     this.bootFrom = (fastForward && fastForward.boot) || null
 
-    // conservative (default on): only fast-forward onto a head someone can serve whole
-    this._conservativeFF = !fastForward || !!fastForward.conservative
+    // conservative (default on): only fast-forward onto a system and view held whole
+    this._conservativeFF = fastForward === null || fastForward.conservative !== false
 
     this.trusted = new TrustedPeers(this, handlers)
 
@@ -828,7 +828,7 @@ module.exports = class Autobee extends ReadyResource {
 
     for (const { key, length } of hints) {
       if (length === 0) continue
-      promises.push(this._resolveOplogHint(key, length, timeout ? { timeout } : null))
+      promises.push(this._resolveOplogHint(key, length, { timeout }))
     }
 
     const ops = await Promise.all(promises)
@@ -867,7 +867,7 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _getOplog(key, length, opts = null) {
+  async _getOplog(key, length, opts) {
     const core = this.openCore(key)
 
     try {
@@ -878,17 +878,12 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   // reads on a core the caller owns, so a walk does not churn a session per step
-  async readOplog(core, length, opts = null) {
+  async readOplog(core, length, { timeout = 0 } = {}) {
     await core.ready()
 
-    const { conservative = false, timeout } = opts === null ? {} : opts
-
     const target = length >= 0 ? length : core.length
     if (target === 0) return null
 
-    // conservative: only proceed when a connected peer can serve the head whole
-    if (conservative && core.remoteContiguousLength < target) return null
-
     const buf = await core.get(target - 1, { timeout })
     if (buf === null) return null
 
@@ -1218,16 +1213,17 @@ module.exports = class Autobee extends ReadyResource {
     return t
   }
 
-  async applyBacklog(batches) {
-    const queue = batches.slice()
+  async _processBatch(batch) {
+    // a stack: the tip is pushed in reverse so it pops in order
+    const stack = [batch]
 
-    while (queue.length) {
-      const batch = queue.shift()
+    while (stack.length) {
+      const batch = stack.pop()
       const t = await this.prepareBatch(batch)
 
       if (t.view) {
         this._workingBee.move(t.view)
-        queue.unshift(...t.tip)
+        for (let i = t.tip.length - 1; i >= 0; i--) stack.push(t.tip[i])
         continue
       }
 
@@ -1237,15 +1233,12 @@ module.exports = class Autobee extends ReadyResource {
 
       await this._applyBatch(batch, batch[0].optimistic)
 
-      this.writers.triggers.trigger(
-        b4a.toString(batch[0].key, 'hex'),
-        batch[batch.length - 1].length
-      )
-    }
-  }
+      // trigger any upstream writer waiting for this batch early
+      const id = b4a.toString(batch[0].key, 'hex')
+      const length = batch[batch.length - 1].length
 
-  async _processBatch(batch) {
-    await this.applyBacklog([batch])
+      this.writers.triggers.trigger(id, length)
+    }
   }
 
   async _applyBatch(batch, optimistic) {

package/lib/fast-forward.js

@@ -10,9 +10,12 @@ const DEFAULT_OP_TIMEOUT = 5_000
 const MIN_FF_GAP = 32
 
 class FastForward {
-  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
+  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT, conservative = false } = {}) {
     this.auto = auto
 
+    // conservative: only land when the system and view we booted are held whole
+    this.conservative = conservative
+
     this.activeRequests = []
 
     // 'reboot' is a storage namespace: renaming it would derive a different
@@ -38,15 +41,11 @@ class FastForward {
 
   static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
     const conservative = !force && auto._conservativeFF
-    const timeoutOpts = timeout ? { timeout } : null
 
-    const oplog = await FastForward.flushHead(auto, head, {
-      conservative,
-      ...timeoutOpts
-    })
+    const oplog = await FastForward.flushHead(auto, head, { timeout })
     if (!oplog) return null
 
-    const verified = trusted ? await FastForward.flushHead(auto, trusted, timeoutOpts) : oplog
+    const verified = trusted ? await FastForward.flushHead(auto, trusted, { timeout }) : oplog
     if (!verified) return null
 
     // legacy nodes from non-indexers have no system info to fast-forward from
@@ -57,7 +56,8 @@ class FastForward {
     }
 
     return new FastForward(auto, batchToHead(verified.op.views.system), {
-      timeout: timeout || DEFAULT_OP_TIMEOUT
+      timeout: timeout || DEFAULT_OP_TIMEOUT,
+      conservative
     })
   }
 
@@ -68,7 +68,7 @@ class FastForward {
 
     for (const head of heads) {
       if (head.length === 0) continue
-      promises.push(FastForward.flushHead(auto, head, timeout ? { timeout } : null))
+      promises.push(FastForward.flushHead(auto, head, { timeout }))
     }
 
     const ops = await Promise.all(promises)
@@ -180,8 +180,10 @@ class FastForward {
     promises.push(this.system.get(this.auto.local.key, { timeout: this.timeout }))
 
     // a legacy system without the designated view migrates onto an empty view
+    let view = null
+
     if (this.system.view.length) {
-      const view = this.auto.store.get({ key: this.system.view.key, active: true })
+      view = this.auto.store.get({ key: this.system.view.key, active: true })
       this.cores.push(view)
 
       promises.push(
@@ -199,6 +201,13 @@ class FastForward {
     await Promise.all(promises)
     if (this.destroyed) return null
 
+    // conservative: the lengths we land on must be held whole, either by us or
+    // by a connected peer, so the view does not stall on a block nobody has
+    if (this.conservative) {
+      if (!isContiguous(this.system.bee.core, this.head.length)) return null
+      if (view !== null && !isContiguous(view, this.system.view.length)) return null
+    }
+
     return {
       head: this.head,
       migrate: null
@@ -360,6 +369,10 @@ async function findNewestSystem(ff, info) {
   return { legacy, autobee }
 }
 
+function isContiguous(core, length) {
+  return core.contiguousLength >= length || core.remoteContiguousLength >= length
+}
+
 function batchToHead(b) {
   return {
     key: b.key,

package/lib/triggers.js

@@ -1,9 +1,14 @@
+// writers waiting on a node of another writer, keyed by that writer's id.
+// a writer waits on at most one entry at a time, tracked as writer.waiting
 module.exports = class Triggers {
   constructor() {
     this.waiting = new Map()
   }
 
   add(id, length, writer) {
+    // a writer only ever waits on one link: drop whatever it waited on before
+    if (writer.waiting !== null && writer.waiting.id !== id) this.remove(writer.waiting)
+
     let w = this.waiting.get(id)
 
     if (!w) {
@@ -24,6 +29,7 @@ module.exports = class Triggers {
   }
 
   remove(entry) {
+    // writer is nulled on removal, so a removed entry is a noop here
     if (!entry.writer) return
 
     const w = this.waiting.get(entry.id)
@@ -42,14 +48,18 @@ module.exports = class Triggers {
     const w = this.waiting.get(id)
     if (!w) return
 
+    // collect first: the walk swap-removes, so nothing may touch the array mid-loop
+    const bump = []
+
     for (let i = w.length - 1; i >= 0; i--) {
       const entry = w[i]
       if (entry.length > length) continue
 
-      const writer = entry.writer
+      bump.push(entry.writer)
       this._remove(w, entry)
-      writer.bump()
     }
+
+    for (const writer of bump) writer.bump()
   }
 
   _remove(w, entry) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.7.1",
+  "version": "2.7.2",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",