autobee 2.0.0 → 2.1.0

package/index.js

@@ -51,11 +51,13 @@ module.exports = class Autobee extends ReadyResource {
     this.getSystemEncryption = this._getEncryptionProvider.bind(this, '_system')
     this.getViewEncryption = this._getEncryptionProvider.bind(this, viewName)
 
-    const bee = new Hyperbee(store.namespace('view'), {
+    const beeStore = store.session()
+    const bee = new Hyperbee(beeStore, {
+      core: beeStore.get({ preload: this._getCorePreload(viewName) }),
       // defer one tick to ensure consistent state, then return state prom
       preload: async () => {
         await 1
-        await this._bootReady()
+        await this._bootGuard.ready()
       },
       getEncryptionProvider: this.getViewEncryption
     })
@@ -69,11 +71,12 @@ module.exports = class Autobee extends ReadyResource {
     this._handlers = handlers
     this.stats = { undos: 0, fastForwards: 0, drains: 0, applies: 0, appends: 0 }
 
-    this.system = new System(this.store.namespace('system'), this.name, {
-      getEncryptionProvider: this.getSystemEncryption,
-      encrypted: this.encrypted
+    const systemStore = this.store.session()
+    this.system = new System(systemStore, this.name, {
+      core: systemStore.get({ preload: this._getCorePreload('system') }),
+      encrypted: this.encrypted,
+      getEncryptionProvider: this.getSystemEncryption
     })
-    this.system.auto = this
 
     this.bee = bee.snapshot()
     this.view = ApplyView.open(this.bee, this)
@@ -120,9 +123,11 @@ module.exports = class Autobee extends ReadyResource {
 
     this.legacyViews = handlers.legacyViews || []
 
+    // the guards may be destroyed before anyone waits on them
     this._bootGuard = new ReadyGuard()
-    this._bootingState = null
-    this._bootingAll = null
+    this._bootGuard.ready().catch(safetyCatch)
+    this._bootOnlineGuard = new ReadyGuard()
+    this._bootOnlineGuard.ready().catch(safetyCatch)
 
     this._now = handlers.now || Date.now // overridable for clock-drift tests
     this._preapply = handlers.preapply || null
@@ -132,7 +137,6 @@ module.exports = class Autobee extends ReadyResource {
     this._needsUpdate = false
     this._approvalCheck = true
     this._approvedPending = new Map()
-    this._approvalRequests = []
     this._prefetchingApprovals = false
     this._updateLocalCore = null
     this._host = new ApplyCalls(this)
@@ -150,6 +154,7 @@ module.exports = class Autobee extends ReadyResource {
 
     this._catchupMigratedNodes = null
     this._migratedHead = null
+    this._prebooting = null
 
     this.ready().catch(noop)
   }
@@ -181,24 +186,56 @@ module.exports = class Autobee extends ReadyResource {
     return !!(this._draining || this._updating)
   }
 
+  async _getCorePreload(name) {
+    await 1
+    const result = await this._prebooting
+    return {
+      name: 'autobee/' + result.local.id + '/' + name,
+      encryption: name === 'system' ? this.getSystemEncryption() : this.getViewEncryption(),
+      inflightRange: [256, 512]
+    }
+  }
+
+  // network free: only a migration needs peers, so ready() awaits the full
+  // state boot unless one is pending and defers it to the background if so -
+  // anything needing the booted system must wait on the boot guard
   async _open() {
-    await this._preBoot()
+    this._prebooting = this._preBoot()
 
-    this._bootingState = this._bootState()
-    this._bootingAll = this._bootAll()
+    let result = null
 
-    this._bootingState.catch(safetyCatch)
-    this._bootingAll.catch(safetyCatch)
+    try {
+      result = await this._prebooting
+
+      this.key = result.key
+      this.bootstrap = result.bootstrap
+      this.discoveryKey = result.bootstrap.core.discoveryKey
+      this.id = result.bootstrap.core.id
+      this.encryptionKey = result.encryptionKey
+      this.previousDrain = result.previousDrain
+      this.local = result.local
+
+      if (this.encrypted && this.encryptionKey === null) {
+        throw new Error('Encryption key is expected')
+      }
 
-    await this.bee.ready()
-    await this._workingBee.ready()
-    await this._bootingState
+      this.local.setEncryption(this._getEncryptionProvider())
+      this.writers = new ActiveWriters(this)
+    } catch (err) {
+      // unblock the guard awaiters (bee preloads, bumps, wakeups, flushes)
+      this._bootGuard.destroy(err)
+      this._bootOnlineGuard.destroy(err)
+      throw err
+    }
 
-    this._localSystemStart = this.system.bee.context.local.length
-    this._localViewStart = this._workingBee.context.local.length
-    this._localFlushes = this.system.flushes
+    const booting = this._boot()
 
-    this.bumpSoon()
+    // waits on the boot guard internally and destroys its own guard on failure
+    this._bootOnline()
+
+    // if migrating, we might need network io, if not lets wait on it - less surprises
+    if (result.migration) booting.catch(this._onErrorBound)
+    else await booting
   }
 
   _requestWakeup() {
@@ -250,43 +287,40 @@ module.exports = class Autobee extends ReadyResource {
 
   async _close() {
     this._interrupting = true
-    Hypercore.destroyRequests(this._approvalRequests, null)
-    if (this._bootWait !== null) this._bootWait.resolve()
-    if (this._notifyHandler) this._notifyHandler.destroy()
-    if (this._draining) {
-      // drain may be waiting on replicated blocks
-      this._clearRequests()
-      await this._draining
-    }
-
-    if (this._updating) await this._updating
 
-    // let in-flight writer adds finish
     try {
-      await this._bootingAll
-    } catch {}
-
-    await ApplyView.close(this.view, this)
+      await ApplyView.close(this.view, this)
+    } catch (err) {
+      safetyCatch(err)
+    }
 
     if (this.writers) await this.writers.close()
     await this.local.close()
     await this.system.close()
     await this._wakeup.close()
-    if (this.bootstrap && !this.bootstrap.closed) await this.bootstrap.close()
-    await this._workingView.close()
-    await this.bee.close()
-    await this.store.close()
-  }
+    if (this.bootstrap) await this.bootstrap.close()
 
-  _clearRequests() {
-    const closingError = new Error('Autobee is closing')
-    for (const session of this.store.sessions) {
-      if (session.opened && !session.closing) {
-        session.clearRequests(session.activeRequests, closingError)
-      }
+    try {
+      await this._workingView.close()
+      await this.bee.close()
+    } catch (err) {
+      safetyCatch(err)
     }
 
-    if (this.fastForwarding) this.fastForwarding.clearRequests(closingError)
+    // don't rely on the store teardown to stop the notify watcher
+    if (this._notifyHandler) this._notifyHandler.destroy()
+
+    // rugpull the rest
+    await this.store.close()
+
+    if (this._bootWait !== null) this._bootWait.resolve()
+    if (this._updating) await this._updating
+    if (this._draining) await this._draining
+
+    // let in-flight writer adds finish
+    try {
+      await this._bootOnlineGuard.ready()
+    } catch {}
   }
 
   replicate(...args) {
@@ -296,7 +330,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async flush() {
-    await this._bootingAll
+    await this._bootOnlineGuard.ready()
   }
 
   hintWakeup(wakeup) {
@@ -322,6 +356,10 @@ module.exports = class Autobee extends ReadyResource {
     return new WriterEncryption(this)
   }
 
+  getMostRecentHead() {
+    return topo.getMostRecentHead(this, this.system.bee.snapshot())
+  }
+
   async _preBoot() {
     if (this._handlers.wait) await this._handlers.wait()
 
@@ -338,17 +376,18 @@ module.exports = class Autobee extends ReadyResource {
     if (this.bootFrom) {
       this.bootFrom = getBootOption(await this.bootFrom)
     }
-  }
 
-  getMostRecentHead() {
-    return topo.getMostRecentHead(this, this.system.bee.snapshot())
+    return boot(this.store, this.key, this.legacyViews, {
+      encryptionKey: this.encryptionKey,
+      keyPair: this.keyPair
+    })
   }
 
-  async _bootState() {
+  async _boot() {
     if (!this._bootGuard.enter()) return this._bootGuard.ready()
 
     try {
-      await this._bootStateUnsafe()
+      await this._bootUnsafe()
     } catch (err) {
       this._bootGuard.destroy(err)
       throw err
@@ -356,40 +395,13 @@ module.exports = class Autobee extends ReadyResource {
 
     this._bootGuard.exit()
 
-    return this._bootGuard.ready()
-  }
+    this.bumpSoon()
 
-  async _bootReady() {
-    if (this._bootGuard.opened) return true
-    try {
-      await this._bootGuard.ready()
-      return true
-    } catch {
-      return false
-    }
+    return this._bootGuard.ready()
   }
 
-  async _bootStateUnsafe() {
-    const result = await boot(this.store, this.key, this.legacyViews, {
-      encryptionKey: this.encryptionKey,
-      keyPair: this.keyPair
-    })
-
-    this.key = result.key
-    this.bootstrap = result.bootstrap
-    this.discoveryKey = result.bootstrap.core.discoveryKey
-    this.id = result.bootstrap.core.id
-    this.encryptionKey = result.encryptionKey
-    this.previousDrain = result.previousDrain
-    this.local = result.local
-
-    if (this.encrypted && this.encryptionKey === null) {
-      throw new Error('Encryption key is expected')
-    }
-
-    this.local.setEncryption(this._getEncryptionProvider())
-
-    this.writers = new ActiveWriters(this)
+  async _bootUnsafe() {
+    const result = await this._prebooting
 
     if (this._handlers.wakeupCapability) {
       this.wakeupCapability = await this._handlers.wakeupCapability
@@ -458,31 +470,53 @@ module.exports = class Autobee extends ReadyResource {
     this.bee.move(view)
 
     await this.writers.updateLocalState()
+
+    // roots are moved, so these resolve without re-entering the boot guard
+    await this.bee.ready()
+    await this._workingBee.ready()
+
+    // baseline before the guard opens so an early drain can't flush against stale offsets
+    this._localSystemStart = this.system.bee.context.local.length
+    this._localViewStart = this._workingBee.context.local.length
+    this._localFlushes = this.system.flushes
   }
 
-  async _bootAll() {
-    if (!(await this._bootReady())) return
+  async _bootOnline() {
+    if (!this._bootOnlineGuard.enter()) return
 
-    for (const head of this.system.heads) {
-      await this.writers.add(head.key)
-    }
+    try {
+      if (!this._bootGuard.opened) await this._bootGuard.ready()
+
+      for (const head of this.system.heads) {
+        await this.writers.add(head.key)
+      }
+
+      if (!this.system.heads.length) {
+        await this.writers.add(this.bootstrap.key)
+      }
 
-    if (!this.system.heads.length) {
-      await this.writers.add(this.bootstrap.key)
+      await this._bump(true)
+    } catch (err) {
+      this._bootOnlineGuard.destroy(err)
+      return
     }
 
-    await this._bump()
+    this._bootOnlineGuard.exit()
   }
 
   bumpSoon() {
-    this._bump().catch(safetyCatch)
+    this._bump(false).catch(safetyCatch)
   }
 
-  async _bump() {
-    if (!(await this._bootReady())) return
+  async _bump(force) {
+    if (!force && !this._bootGuard.opened) await this._bootGuard.ready()
 
+    // resolve before the bootAll gate: a parked boot-from retries on any bump,
+    // and its drain is the one _bootOnline itself is waiting on
     if (this._bootWait !== null) this._bootWait.resolve()
 
+    if (!force && !this._bootOnlineGuard.opened) await this._bootOnlineGuard.ready()
+
     this.bumping++
 
     if (!this._draining) {
@@ -493,11 +527,11 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   update() {
-    return this._bump()
+    return this._bump(false)
   }
 
   async updated() {
-    if (this.opened === false) await this.ready()
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
     if (this._draining) return this._draining
     return Promise.resolve()
   }
@@ -796,7 +830,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async setLocal(key, { keyPair } = {}) {
-    if (!this.opened) await this.ready()
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
     if (this.closing) throw new Error('Autobee closed')
 
     const manifest = keyPair
@@ -925,15 +959,13 @@ module.exports = class Autobee extends ReadyResource {
     if (this._prefetchingApprovals) return
     this._prefetchingApprovals = true
 
-    const activeRequests = this._approvalRequests
-
     try {
       const prefetch = []
-      const standing = await this._standing(activeRequests)
+      const standing = await this._standing()
       if (standing <= 0 || !this._pendingWork(standing)) return
-      for await (const p of this._servableRequests(standing, activeRequests)) {
-        prefetch.push(this.system.get(p.key, { activeRequests }).catch(safetyCatch))
-        prefetch.push(this.system.grantHint(p.key, { activeRequests }).catch(safetyCatch))
+      for await (const p of this._servableRequests(standing)) {
+        prefetch.push(this.system.get(p.key).catch(safetyCatch))
+        prefetch.push(this.system.grantHint(p.key).catch(safetyCatch))
       }
       await Promise.allSettled(prefetch)
     } finally {
@@ -941,8 +973,8 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _standing(activeRequests) {
-    const rec = await this.system.get(this.local.key, { activeRequests })
+  async _standing() {
+    const rec = await this.system.get(this.local.key)
     return currentWeight(rec)
   }
 
@@ -960,8 +992,8 @@ module.exports = class Autobee extends ReadyResource {
 
   // the requests we could serve and by how much - shared by the prefetch and
   // the collector so the two cannot drift on what counts as ours
-  async *_servableRequests(standing, activeRequests) {
-    for await (const p of this.system.listPendingPromotions({ activeRequests })) {
+  async *_servableRequests(standing) {
+    for await (const p of this.system.listPendingPromotions()) {
       const amount = Math.min(standing, p.weight)
       if (amount <= 0) continue
       yield { key: p.key, amount }
@@ -972,8 +1004,7 @@ module.exports = class Autobee extends ReadyResource {
     if (!this.system.promotions.changed && !this._approvalCheck) return null
     if (!this.writers.writable) return null
 
-    const activeRequests = this._approvalRequests
-    const standing = await this._standing(activeRequests)
+    const standing = await this._standing()
     if (standing <= 0) return null
 
     this.system.promotions.changed = false
@@ -984,7 +1015,7 @@ module.exports = class Autobee extends ReadyResource {
     const approvals = []
     const approving = []
 
-    for await (const p of this._servableRequests(standing, activeRequests)) {
+    for await (const p of this._servableRequests(standing)) {
       approving.push(fetchApproval.call(this, p.key, p.amount, approvals))
     }
 
@@ -1008,10 +1039,7 @@ module.exports = class Autobee extends ReadyResource {
     async function fetchApproval(key, weight, approvals) {
       const hex = b4a.toString(key, 'hex')
       if ((this._approvedPending.get(hex) || 0) >= weight) return
-      const [target, hint] = await Promise.all([
-        this.system.get(key, { activeRequests }),
-        this.system.grantHint(key, { activeRequests })
-      ])
+      const [target, hint] = await Promise.all([this.system.get(key), this.system.grantHint(key)])
       if (!target || target.isRemoved) return
       if (hint && hint.weight >= weight) return
       approvals.push({ key, weight })
@@ -1209,10 +1237,12 @@ module.exports = class Autobee extends ReadyResource {
     return encoding.encodeValue(value, opts)
   }
 
+  // gates on the state boot only, so hints can still feed a parked boot-from -
+  // the trailing bump waits for bootAll like any other drain
   async wakeup({ key, length }) {
-    if (!(await this._bootReady())) return
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
     await this.writers.wakeup(key, length)
-    await this._bump()
+    await this._bump(false)
   }
 
   async append(values, { optimistic = false } = {}) {
@@ -1222,7 +1252,7 @@ module.exports = class Autobee extends ReadyResource {
 
     if (!Array.isArray(values)) values = [values]
 
-    if (!this.opened) await this.ready()
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
 
     if (!optimistic && this.writers.localWriter.isRemoved) {
       throw new Error('Not writable')
@@ -1263,7 +1293,7 @@ module.exports = class Autobee extends ReadyResource {
       batch.push(node)
     }
 
-    return this._bump()
+    return this._bump(false)
   }
 
   async _flushLocal() {
@@ -1294,7 +1324,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async moveTo(head) {
-    if (!this.opened) await this.ready()
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
     if (this.closing) throw new Error('Autobee closed')
 
     const ff = await FastForward.fromHead(this, head, null, { force: true })

package/lib/system.js

@@ -8,8 +8,6 @@ const topo = require('./topo.js')
 
 const INFO_KEY = b4a.from([0])
 const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
-const INDEXER_GTE = b4a.from([2])
-const INDEXER_LT = b4a.from([3])
 const EMPTY_HEAD = { length: 0, key: null }
 const EMPTY = b4a.from([])
 
@@ -124,13 +122,6 @@ module.exports = class Systembee {
     return null
   }
 
-  async *getIndexers() {
-    for await (const data of this.bee.createReadStream({ gte: INDEXER_GTE, lt: INDEXER_LT })) {
-      const key = data.key.subarray(1)
-      yield key
-    }
-  }
-
   async reset({ timeout } = {}) {
     const info = await this.getInfo({ timeout })
 

package/lib/topo.js

@@ -76,7 +76,7 @@ async function sort(auto, batch) {
   for await (const data of ch) {
     if (isLegacyNode(data)) break
 
-    const { ref } = getOplog(data)
+    const ref = getOplog(data)
     const link = {
       key: ref.oplog.key,
       length: ref.oplog.length,
@@ -318,23 +318,18 @@ async function* oplogIterator(bee) {
 }
 
 function getOplog(data) {
-  const result = { ref: null, info: null }
   for (const { keys } of data.batch) {
+    if (!keys) continue
+
     for (const k of keys) {
       const prefix = k.key[0]
 
-      if (prefix === 0) {
-        result.info = encoding.decodeSystemInfo(k.value)
-        if (result.ref) return result
-      }
-
       if (prefix === 1) {
         // expect inlined for now
         const value = encoding.decodeSystemWriter(k.key, k.value)
         if (!value.isOplog) continue
 
-        result.ref = { undo: data.tail || EMPTY_HEAD, oplog: value }
-        if (result.info) return result
+        return { undo: data.tail || EMPTY_HEAD, oplog: value }
       }
     }
   }
@@ -395,6 +390,7 @@ function isLinkingAll(node, heads) {
 
 function isLegacyNode(data) {
   for (const { keys } of data.batch) {
+    if (!keys) continue
     for (const k of keys) {
       if (b4a.equals(k.key, INFO_LEGACY_KEY)) return true
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",