autobee 2.10.1 → 2.11.0

package/index.js

@@ -5,6 +5,7 @@ const safetyCatch = require('safety-catch')
 const Hyperbee = require('hyperbee2')
 const ID = require('hypercore-id-encoding')
 const rrp = require('resolve-reject-promise')
+const Signal = require('signal-promise')
 const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('autobee-encryption')
 const AutobeeWakeup = require('autobee-wakeup')
 const Hypercore = require('hypercore')
@@ -12,6 +13,7 @@ const crypto = require('hypercore-crypto')
 const c = require('compact-encoding')
 const asserts = require('./lib/asserts.js')
 const boot = require('./lib/boot.js')
+const { DEFAULT_MANIFEST_VERSION } = require('./lib/constants.js')
 const { resolveWeight, currentWeight } = require('./lib/witness.js')
 const encoding = require('./lib/encoding.js')
 const FastForward = require('./lib/fast-forward.js')
@@ -26,6 +28,7 @@ const migrations = require('./lib/migrations.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const DEFAULT_ACK_THRESHOLD = 32
+const BOOT_NETWORK_TIMEOUT = 5000
 const INTERRUPT = new Error('Apply interrupted')
 
 module.exports = class Autobee extends ReadyResource {
@@ -116,6 +119,9 @@ module.exports = class Autobee extends ReadyResource {
     this.fastForwardTo = null
     this._ffCandidates = new Map()
     this._ffSearching = null
+    this._bumpSignal = new Signal()
+    this._networkBooted = false
+    this._networkBooting = false
 
     this._workingBee = bee
     this._workingView = new ApplyView(this._workingBee, this)
@@ -143,6 +149,7 @@ module.exports = class Autobee extends ReadyResource {
     this._now = handlers.now || Date.now // overridable for clock-drift tests
     this._preapply = handlers.preapply || null
     this._preApplied = false
+    this._reindexed = false
     this._warmup = handlers.warmup || null
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
@@ -205,7 +212,8 @@ module.exports = class Autobee extends ReadyResource {
     await 1
     const result = await this._prebooting
     return {
-      name: 'autobee/' + result.local.id + '/' + name,
+      name: 'autobee/' + result.local.id + '/view/' + name,
+      manifestVersion: DEFAULT_MANIFEST_VERSION,
       encryption: name === 'system' ? this.getSystemEncryption() : this.getViewEncryption(),
       inflightRange: [256, 512]
     }
@@ -380,6 +388,7 @@ module.exports = class Autobee extends ReadyResource {
 
   async _close() {
     this._interrupting = true
+    this._bumpSignal.notify()
 
     // the local core holds the exclusive lock but the local writer closes it
     // early in the teardown, so hand the lock to a detached session that
@@ -583,6 +592,11 @@ module.exports = class Autobee extends ReadyResource {
     this._localFlushes = this.system.flushes
   }
 
+  async _isReindexing() {
+    if (this._networkBooted || !this._ffEnabled || this._migrating) return false
+    return !(await this._isReindexed())
+  }
+
   async _bootOnline() {
     if (!this._bootOnlineGuard.enter()) return
 
@@ -607,6 +621,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   bumpSoon() {
+    this._bumpSignal.notify()
     this._bump(false).catch(safetyCatch)
   }
 
@@ -668,6 +683,58 @@ module.exports = class Autobee extends ReadyResource {
     this.emit('error', err)
   }
 
+  async compactMaybe() {
+    if (!this.writers.writable) return 0
+
+    if (await this._isReindexed()) return 0
+
+    const view = await this._reindexBee(this._workingBee)
+    if (view > 0) this.bee.move(this._workingBee.head())
+
+    const system = await this._reindexBee(this.system.bee)
+
+    if (view + system > 0) this._reindexed = true
+
+    return view + system
+  }
+
+  async _isReindexed() {
+    const head = this.system.bee.head()
+    if (head === null) return true
+    if (await this._shouldReindex(head.key)) return false
+
+    const view = this.system.view
+    if (!view || !view.key) return true
+    return !(await this._shouldReindex(view.key))
+  }
+
+  async _reindexBee(bee) {
+    const head = bee.head()
+    const local = bee.context.local
+    if (head === null) return 0
+    if ((await this._shouldReindex(local.key)) && local.length > 0) return 0
+    if (!(await this._shouldReindex(head.key))) return 0
+
+    return bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
+  }
+
+  async _shouldReindex(key, { unknown = false, length = 0, timeout = 0 } = {}) {
+    const core = this.store.get({ key, active: false })
+
+    try {
+      await core.ready()
+
+      if (core.manifest === null && length > 0) {
+        await core.get(length - 1, { raw: true, timeout }).catch(safetyCatch)
+      }
+
+      if (core.manifest === null) return unknown
+      return core.manifest.version > 1 && core.manifest.version < DEFAULT_MANIFEST_VERSION
+    } finally {
+      await core.close()
+    }
+  }
+
   // one-shot user gate: nothing applies until the host has resolved whatever
   // state apply depends on (e.g. legacy views recorded by a migration)
   async _runPreApply() {
@@ -688,6 +755,7 @@ module.exports = class Autobee extends ReadyResource {
       const { head = null, legacy = null } = this.bootFrom
 
       this.bootFrom = null
+      this._networkBooted = true
 
       if (legacy) {
         await this._bootFromSystem(legacy)
@@ -695,8 +763,14 @@ module.exports = class Autobee extends ReadyResource {
         this._wakeup.hint({ key: head.key, length: head.length || 0 })
         await this._bootFromHead(head)
       }
+    } else if (await this._isReindexing()) {
+      this._networkBooted = true
+      await this._bootFromNetwork()
     }
 
+    // preferably get a peer's compacted view during bootFrom
+    if (this.fastForwardTo === null) await this.compactMaybe()
+
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
@@ -865,6 +939,7 @@ module.exports = class Autobee extends ReadyResource {
         const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
 
         const ff = await FastForward.fromHeads(this, heads, {
+          skipGap: this._networkBooting,
           timeout: FastForward.DEFAULT_TIMEOUT
         })
 
@@ -1208,12 +1283,15 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _appendAck() {
-    if (!this._acking) return false
+    if (!this._acking && !this._reindexed) return false
     if (!this.writers.writable) return false
 
     if (this.writers.localWriter.pending !== null) return false
-    if ((await this._flushesBehind()) < this._ackThreshold) return false
-    if (await this._allHeadsTrusted()) return false
+
+    if (!this._reindexed) {
+      if ((await this._flushesBehind()) < this._ackThreshold) return false
+      if (await this._allHeadsTrusted()) return false
+    }
 
     const links = this.system.getLinks(this.local.key)
     const now = this._now()
@@ -1371,6 +1449,14 @@ module.exports = class Autobee extends ReadyResource {
 
     const changed = await this.system.flush(batch, this._workingBee)
 
+    if (this._reindexed && (await this._isReindexed())) {
+      this._reindexed = false
+      await this.local.setUserData(
+        'autobee/head',
+        encoding.encodeBootRecord(this.system.bootRecord())
+      )
+    }
+
     if (this.system.promotions.changed) this._prefetchApprovals().catch(safetyCatch)
 
     if (local) {
@@ -1551,6 +1637,41 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
+  async _bootFromNetwork() {
+    this._networkBooting = true
+
+    try {
+      await this._waitForNetworkBoot()
+    } finally {
+      this._networkBooting = false
+    }
+  }
+
+  async _waitForNetworkBoot() {
+    const deadline = Date.now() + BOOT_NETWORK_TIMEOUT
+
+    this._requestWakeup()
+
+    while (!this._interrupting && !this.closing && this.fastForwardTo === null) {
+      const hints = await this._applyWakeupHints()
+      if (hints.length) this._queueFastForward(hints)
+
+      const remaining = deadline - Date.now()
+      if (remaining <= 0 || this.fastForwardTo !== null) return
+
+      if (this._ffSearching !== null) {
+        await Promise.race([this._ffSearching, this._bumpSignal.wait(remaining)])
+        this._bumpSignal.notify()
+        if (this._ffSearching === null) return
+        continue
+      }
+
+      if (hints.length) return
+
+      await this._bumpSignal.wait(remaining)
+    }
+  }
+
   async _runFastForward(ff, { force = false } = {}) {
     if (this.fastForwardTo !== null || this.fastForwarding !== null) {
       await ff.close()

package/lib/constants.js

@@ -1,10 +1,12 @@
 const LEGACY_OPLOG_VERSION = 2
 const OPLOG_VERSION = 4
+const DEFAULT_MANIFEST_VERSION = 3
 const LEGACY_AUTOBASE_VERSION = 2
 const AUTOBEE_VERSION = 3
 const DEFAULT_OP_TIMEOUT = 5000
 
 module.exports = {
+  DEFAULT_MANIFEST_VERSION,
   LEGACY_OPLOG_VERSION,
   OPLOG_VERSION,
   LEGACY_AUTOBASE_VERSION,

package/lib/fast-forward.js

@@ -7,7 +7,7 @@ const migrations = require('./migrations.js')
 const { LEGACY_AUTOBASE_VERSION, LEGACY_OPLOG_VERSION } = require('./constants.js')
 
 const DEFAULT_OP_TIMEOUT = 5_000
-const MIN_FF_GAP = 32
+const MIN_FF_GAP = 16
 
 class FastForward {
   constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT, conservative = false } = {}) {
@@ -39,7 +39,7 @@ class FastForward {
 
   static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
 
-  static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
+  static async fromHead(auto, head, trusted, { force = false, skipGap = false, timeout = 0 } = {}) {
     const conservative = !force && auto._conservativeFF
 
     const oplog = await FastForward.flushHead(auto, head, { timeout })
@@ -51,7 +51,7 @@ class FastForward {
     // legacy nodes from non-indexers have no system info to fast-forward from
     if (!oplog.op.views || !verified.op.views) return null
 
-    if (!force && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
+    if (!force && !skipGap && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
       return null
     }
 
@@ -61,7 +61,7 @@ class FastForward {
     })
   }
 
-  static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
+  static async fromHeads(auto, heads, { force = false, skipGap = false, timeout = 0 } = {}) {
     const reference = auto._workingView.view
 
     const promises = []
@@ -71,14 +71,12 @@ class FastForward {
       promises.push(FastForward.flushHead(auto, head, { timeout }))
     }
 
-    const ops = await Promise.all(promises)
+    const ops = await FastForward.reindexedCandidates(auto, await Promise.all(promises), {
+      timeout: timeout || DEFAULT_OP_TIMEOUT
+    })
     if (auto.fastForwarding || auto.fastForwardTo) return null
 
-    const trust = await Promise.all(
-      ops.map((res) => {
-        return res === null ? false : auto.trusted.isTrusted(res.key, reference)
-      })
-    )
+    const trust = await Promise.all(ops.map((res) => auto.trusted.isTrusted(res.key, reference)))
     if (auto.fastForwarding || auto.fastForwardTo) return null
 
     const candidates = []
@@ -88,8 +86,7 @@ class FastForward {
 
     for (let i = 0; i < ops.length; i++) {
       const res = ops[i]
-      if (res === null || !res.op.views) continue
-      if (!force && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
+      if (!force && !skipGap && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
 
       if (!trust[i]) {
         candidates.push(res)
@@ -103,7 +100,7 @@ class FastForward {
     }
 
     if (bestTrusted !== null) {
-      return FastForward.fromHead(auto, bestTrusted, null, { force, timeout })
+      return FastForward.fromHead(auto, bestTrusted, null, { force, skipGap, timeout })
     }
 
     candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
@@ -113,13 +110,38 @@ class FastForward {
       if (trusted === null) continue
       if (auto.fastForwarding || auto.fastForwardTo) return null
 
-      const ff = await FastForward.fromHead(auto, res, trusted, { force, timeout })
+      const ff = await FastForward.fromHead(auto, res, trusted, { force, skipGap, timeout })
       if (ff !== null) return ff
     }
 
     return null
   }
 
+  static async reindexedCandidates(auto, ops, { timeout }) {
+    const withViews = ops.filter((res) => res !== null && res.op.views)
+    const reindexed = await Promise.all(
+      withViews.map((res) => FastForward.isReindexed(auto, res, timeout))
+    )
+    return withViews.filter((res, i) => reindexed[i])
+  }
+
+  static async isReindexed(auto, res, timeout) {
+    const { system, view } = res.op.views
+    const checks = [system, view]
+      .filter((v) => !!v)
+      .map((range) => {
+        const { key, length } = batchToHead(range)
+        if (length === 0) return false
+        return auto._shouldReindex(key, { unknown: true, length, timeout })
+      })
+
+    for (const shouldReindex of await Promise.all(checks)) {
+      if (shouldReindex) return false
+    }
+
+    return true
+  }
+
   static async flushHead(auto, head, opts) {
     const core = auto.openCore(head.key)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.10.1",
+  "version": "2.11.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -27,8 +27,8 @@
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
     "compact-encoding": "^3.3.0",
-    "hyperbee2": "^2.16.0",
-    "hypercore": "^11.33.2",
+    "hyperbee2": "^2.17.0",
+    "hypercore": "^11.37.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.26.2",
     "read-write-mutexify": "^2.1.0",
@@ -36,6 +36,7 @@
     "ready-resource": "^1.2.0",
     "resolve-reject-promise": "^1.1.0",
     "safety-catch": "^1.0.3",
+    "signal-promise": "^1.0.3",
     "sodium-native": "^5.1.0"
   },
   "devDependencies": {