autobee 2.3.1 → 2.4.0

package/README.md

@@ -81,6 +81,7 @@ Options:
   optimistic: true,              // allow optimistic writes from unknown writers
   isTrusted (key, reference) {}, // do we trust this writer, see Fast-forward
   mostRecentTrusted (target, reference) {}, // the head we vouch for, see Fast-forward
+  ackThreshold: 32,              // flushes we may fall behind before acking, see Acking
   fastForward: {}                // see Fast-forward
 }
 ```
@@ -162,6 +163,10 @@ Hint that a new writer core is available at `key` with at least `length` entries
 
 Rotate the local writer to a different key. The new writer takes over as the active oplog.
 
+#### `db.setAcking(acking, [options])`
+
+Override acking, which is otherwise driven by `isTrusted`, see Acking. `options.threshold` defaults to the current threshold.
+
 #### `views = db.views()`
 
 Returns the current system and view core positions. Used for replication coordination.
@@ -270,6 +275,14 @@ Fast-forward onto an oplog head, ignoring the usual distance and conservative ch
 
 Pass `{ timeout }` to bound the reads, so a head nobody can serve fails instead of hanging.
 
+### Acking
+
+Peers fast-forward onto the heads of writers they trust, so a trusted writer that has nothing to say still has to stamp its view progress into its own oplog.
+
+Acking does that: an empty node is appended whenever the local writer falls `ackThreshold` (default `64`) flushes behind the system.
+
+It is enabled for exactly the writers `isTrusted` accepts. The local key is judged when the db opens, after a fast-forward, and after the local writer rotates, so a writer that becomes trusted starts acking without being told to. `setAcking` overrides it until the next of those points.
+
 ### Encryption
 
 Pass an `encryptionKey` to encrypt all writer cores and the view at rest.

package/index.js

@@ -25,6 +25,7 @@ const UpdateChanges = require('./lib/updates.js')
 const migrations = require('./lib/migrations.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
+const DEFAULT_ACK_THRESHOLD = 32
 const INTERRUPT = new Error('Apply interrupted')
 
 module.exports = class Autobee extends ReadyResource {
@@ -112,6 +113,9 @@ module.exports = class Autobee extends ReadyResource {
     this._localSystemStart = 0
     this._localSystemLength = 0
     this._localFlushes = 0
+    this._acking = false
+    this._ackThreshold = handlers.ackThreshold || DEFAULT_ACK_THRESHOLD
+    this._ackFlushes = -1
     this._localViewStart = 0
     this._localViewLength = 0
 
@@ -193,6 +197,18 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
+  setAcking(acking, { threshold = this._ackThreshold } = {}) {
+    this._acking = acking !== false && threshold > 0
+    this._ackThreshold = threshold
+
+    if (this._acking) this.bumpSoon()
+  }
+
+  async _updateAcking() {
+    if (this._interrupting) return
+    this.setAcking(await this.trusted.isTrusted(this.local.key, this.view))
+  }
+
   // network free: only a migration needs peers, so ready() awaits the full
   // state boot unless one is pending and defers it to the background if so -
   // anything needing the booted system must wait on the boot guard
@@ -468,6 +484,8 @@ module.exports = class Autobee extends ReadyResource {
 
     this._bootGuard.exit()
 
+    await this._updateAcking()
+
     this.bumpSoon()
 
     return this._bootGuard.ready()
@@ -672,7 +690,12 @@ module.exports = class Autobee extends ReadyResource {
             break // revaluate conditions...
           }
 
-          if (!(await this._bumpPendingWriters())) break
+          if (await this._bumpPendingWriters()) {
+            this._needsUpdate = true
+            continue
+          }
+
+          if (!(await this._appendAck())) break
           this._needsUpdate = true
         }
 
@@ -922,6 +945,7 @@ module.exports = class Autobee extends ReadyResource {
     await this.writers.rotateLocalWriter(this.local)
 
     this._updateLocalCore = null
+    this._ackFlushes = -1
 
     this.local.setUserData('referrer', this.key)
     if (this.encryptionKey) {
@@ -931,6 +955,8 @@ module.exports = class Autobee extends ReadyResource {
     await this.bootstrap.setUserData('autobee/local', this.local.key)
     await oldLocal.close()
 
+    await this._updateAcking()
+
     // done, soft restart
     this.emit('rotate-local-writer')
   }
@@ -1101,6 +1127,30 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
+  async _appendAck() {
+    if (!this._acking) return false
+    if (!this.writers.writable) return false
+
+    if (this.writers.localWriter.pending !== null) return false
+
+    if ((await this._flushesBehind()) < this._ackThreshold) return false
+
+    const links = this.system.getLinks(this.local.key)
+    const t = Math.max(this._now(), this.system.timestamp)
+
+    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null)
+    return true
+  }
+
+  async _flushesBehind() {
+    if (this._ackFlushes === -1) {
+      const latest = await this.writers.getLatestLocalOplog()
+      this._ackFlushes = latest && latest.views ? latest.views.flushes : this.system.flushes
+    }
+
+    return this.system.flushes - this._ackFlushes
+  }
+
   async _bumpPendingWriters({ local = false } = {}) {
     if (!local && this._catchupMigratedNodes !== null) {
       const updated = await this._bumpMigratedWriters()
@@ -1358,7 +1408,7 @@ module.exports = class Autobee extends ReadyResource {
       this._needsUpdate = true
     }
 
-    await this.writers.flushLocal({
+    const flushed = await this.writers.flushLocal({
       flushes: this._localFlushes,
       system: {
         key: this.system.bee.context.local.key,
@@ -1372,6 +1422,8 @@ module.exports = class Autobee extends ReadyResource {
       }
     })
 
+    if (flushed) this._ackFlushes = this._localFlushes
+
     this._localSystemStart = this.system.bee.context.local.length
     this._localSystemLength = 0
     this._localFlushes = this.system.flushes
@@ -1481,6 +1533,8 @@ module.exports = class Autobee extends ReadyResource {
     await this._update(changes, true)
     await this._storeBoot()
 
+    await this._updateAcking()
+
     this.stats.fastForwards++
     this.emit('move-to', to, from)
     this.ff.resolve({ to, from })

package/lib/fast-forward.js

@@ -1,13 +1,15 @@
 const safetyCatch = require('safety-catch')
+const b4a = require('b4a')
 const Hypercore = require('hypercore')
 
 const System = require('./system.js')
 const migrations = require('./migrations.js')
+const { LEGACY_AUTOBASE_VERSION, LEGACY_OPLOG_VERSION } = require('./constants.js')
 
 const DEFAULT_OP_TIMEOUT = 5_000
 const MIN_FF_GAP = 32
 
-module.exports = class FastForward {
+class FastForward {
   constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
     this.auto = auto
 
@@ -162,6 +164,10 @@ module.exports = class FastForward {
 
     await this.system.boot(this.head, { timeout: this.timeout })
 
+    if (this.system.version <= LEGACY_AUTOBASE_VERSION) {
+      if (!(await this._runLegacy())) return null
+    }
+
     const promises = []
 
     // ensure local key is locally available always
@@ -202,6 +208,68 @@ module.exports = class FastForward {
     return migrations.coreLength(core, this.timeout)
   }
 
+  async _runLegacy() {
+    const resolved = await this._resolveLegacy()
+    if (resolved === null) return false
+
+    const { head, autobee } = resolved
+
+    if (autobee !== null) {
+      try {
+        this.head = batchToHead(autobee.op.views.system)
+        await this.system.boot(this.head, { timeout: this.timeout })
+        if (this.system.version > LEGACY_AUTOBASE_VERSION) return true
+      } catch (err) {
+        safetyCatch(err)
+      }
+    }
+
+    this.head = head
+    await this.system.boot(this.head, { timeout: this.timeout })
+
+    return true
+  }
+
+  async _resolveLegacy() {
+    let head = { key: this.head.key, length: this.head.length }
+    const seen = new Set()
+
+    // bound at 5 hops for safety
+    let hops = 5
+    while (true) {
+      seen.add(b4a.toString(head.key, 'hex'))
+
+      const core = this.auto.store.get({ key: head.key, encryption: null })
+      this.cores.push(core)
+      await core.ready()
+
+      await core.setEncryption(this.auto.getSystemEncryption())
+
+      // successors are resolved by key only - wait for the replicated length
+      if (!head.length) head.length = await migrations.coreLength(core, this.timeout)
+      if (!head.length || this.destroyed) return null
+
+      const info = await migrations.readLegacySystemInfo(core, head.length, {
+        timeout: this.timeout,
+        activeRequests: this.activeRequests
+      })
+
+      const { legacy, autobee } = await findNewestSystem(this, info)
+      if (this.destroyed) return null
+
+      // an autobee candidate ends the crawl: it may be behind, but once we are
+      // on autobee the ordinary fast-forward carries us to the newest view
+      if (autobee !== null) return { head, autobee }
+
+      if (hops-- > 0 && legacy !== null && !seen.has(b4a.toString(legacy, 'hex'))) {
+        head = { key: legacy, length: 0 }
+        continue
+      }
+
+      return { head, autobee: null }
+    }
+  }
+
   // cancels every read in flight and fails any that come after
   clearRequests(err = null) {
     Hypercore.destroyRequests(this.activeRequests, err)
@@ -215,6 +283,50 @@ module.exports = class FastForward {
   }
 }
 
+module.exports = FastForward
+
+async function findNewestSystem(ff, info) {
+  let autobee = null
+  let legacy = null
+  let flushes = -1
+
+  for (const idx of info.indexers) {
+    const core = ff.auto.openCore(idx.key)
+    ff.cores.push(core)
+    await core.ready()
+
+    const length = await migrations.coreLength(core, ff.timeout)
+    if (!length || length <= idx.length || ff.destroyed) continue
+
+    const oplog = await FastForward.flushHead(
+      ff.auto,
+      { key: idx.key, length },
+      { timeout: ff.timeout }
+    )
+    if (oplog === null || !oplog.op.views) continue
+
+    // this indexer has already moved to autobee: follow it rather than
+    // digging a legacy stamp out of its history
+    if (oplog.op.version > LEGACY_OPLOG_VERSION) {
+      if (autobee === null || oplog.op.views.flushes > autobee.op.views.flushes) {
+        autobee = oplog
+      }
+      continue
+    }
+
+    const sys = oplog.op.views.system
+    if (!sys || !sys.key) continue
+
+    // follow whichever indexer has checkpointed furthest
+    if (oplog.op.views.flushes > flushes) {
+      flushes = oplog.op.views.flushes
+      legacy = sys.key
+    }
+  }
+
+  return { legacy, autobee }
+}
+
 function batchToHead(b) {
   return {
     key: b.key,

package/lib/migrations.js

@@ -21,6 +21,7 @@ module.exports = {
   getCoreManifest,
   getViewMap,
   checkAutobaseMigration,
+  readLegacySystemInfo,
   inflateLegacyOplog,
   coreLength
 }

package/lib/writers.js

@@ -122,11 +122,12 @@ class ActiveWriters {
     const w = this.localWriter
 
     // only consume the queue when a flush will actually write
-    if (w.pending === null || w.processed === 0) return
+    if (w.pending === null || w.processed === 0) return false
 
     const trusted = await this.auto.trusted.mostRecentTrusted(this.auto._workingView.view, null)
 
-    return w.flush(views, trusted)
+    await w.flush(views, trusted)
+    return true
   }
 
   async refresh() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",