autobee 1.0.3 → 1.0.5

package/index.js

@@ -1,8 +1,10 @@
 const ReadyResource = require('ready-resource')
+const ReadyGuard = require('ready-guard')
 const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const Hyperbee = require('hyperbee2')
 const ID = require('hypercore-id-encoding')
+const rrp = require('resolve-reject-promise')
 const { AutobeeEncryption, WriterEncryption } = require('autobee-encryption')
 const AutobeeWakeup = require('autobee-wakeup')
 const Hypercore = require('hypercore')
@@ -11,6 +13,7 @@ const c = require('compact-encoding')
 const asserts = require('./lib/asserts.js')
 const boot = require('./lib/boot.js')
 const encoding = require('./lib/encoding.js')
+const FastForward = require('./lib/fast-forward.js')
 const System = require('./lib/system.js')
 const ApplyCalls = require('./lib/apply-calls.js')
 const topo = require('./lib/topo.js')
@@ -19,6 +22,7 @@ const UpdateChanges = require('./lib/updates.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const INTERRUPT = new Error('Apply interrupted')
+const MIN_FF_GAP = 32
 
 module.exports = class Autobee extends ReadyResource {
   constructor(store, key = null, handlers = {}) {
@@ -32,14 +36,15 @@ module.exports = class Autobee extends ReadyResource {
     const { name = null, encrypted, encryptionKey } = handlers
 
     this.encrypted = encrypted === true || !!encryptionKey
+    this._getEncryptionProviderBound = this._getEncryptionProvider.bind(this)
 
     const bee = new Hyperbee(store.namespace('view'), {
       // defer one tick to ensure consistent state, then return state prom
       preload: async () => {
         await 1
-        await this._bootingState
+        if (!this._bootGuard.opened) await this._bootGuard.ready()
       },
-      getEncryptionProvider: () => this._getEncryptionProvider()
+      getEncryptionProvider: this._getEncryptionProviderBound
     })
 
     this.store = store
@@ -50,7 +55,7 @@ module.exports = class Autobee extends ReadyResource {
     this.bootstrap = null
 
     this.system = new System(this.store.namespace('system'), this.name, {
-      getEncryptionProvider: () => this._getEncryptionProvider(),
+      getEncryptionProvider: this._getEncryptionProviderBound,
       encrypted: this.encrypted
     })
 
@@ -66,12 +71,18 @@ module.exports = class Autobee extends ReadyResource {
     this.writers = null
     this.bumping = 0
 
+    this.fastForwardBoot = handlers.fastForward || null
+    this.fastForward = null
+    this.fastForwarding = null
+    this.fastForwardTo = null
+
     this._workingBee = bee
     this._workingView = handlers.open ? handlers.open(this._workingBee, this) : this._workingBee
 
     this._appending = []
     this._draining = null
 
+    this._bootGuard = new ReadyGuard()
     this._bootingState = null
     this._bootingAll = null
 
@@ -81,13 +92,16 @@ module.exports = class Autobee extends ReadyResource {
     this._needsUpdate = false
     this._updateLocalCore = null
     this._host = new ApplyCalls(this)
+    this._notifyHandler = null
 
     this.interrupted = null
     this._interrupting = false
     this._onErrorBound = this._onError.bind(this)
+    this._bumpSoonBound = this.bumpSoon.bind(this)
 
-    this._wakeup = new AutobeeWakeup(this, handlers)
     this.wakeupCapability = null
+    this._wakeup = new AutobeeWakeup(this, handlers)
+    this.previousDrain = 0
 
     this.ready().catch(noop)
   }
@@ -127,6 +141,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   _registerWakeup() {
+    if (!this.wakeupCapability) return
     this._wakeup.recouple()
     this._wakeup.setCapability(this.wakeupCapability.key, this.wakeupCapability.discoveryKey)
   }
@@ -140,7 +155,7 @@ module.exports = class Autobee extends ReadyResource {
     return keys
   }
 
-  async getWriterViews(key) {
+  getWriterViews(key) {
     const id = b4a.toString(key, 'hex')
     const w = this.writers.active.get(id)
     if (!w) return []
@@ -160,6 +175,7 @@ module.exports = class Autobee extends ReadyResource {
 
   async _close() {
     this._interrupting = true
+    if (this._notifyHandler) this._notifyHandler.destroy()
     if (this._draining) await this._draining
 
     if (this._handlers.close) await this._handlers.close(this.view)
@@ -215,6 +231,8 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _bootState() {
+    if (!this._bootGuard.enter()) return this._bootGuard.ready()
+
     const result = await boot(this.store, this.key, {
       encryptionKey: this.encryptionKey,
       keyPair: this.keyPair
@@ -225,6 +243,7 @@ module.exports = class Autobee extends ReadyResource {
     this.discoveryKey = result.bootstrap.core.discoveryKey
     this.id = result.bootstrap.core.id
     this.encryptionKey = result.encryptionKey
+    this.previousDrain = result.previousDrain
 
     if (this.encrypted) {
       asserts.assert(this.encryptionKey !== null, 'Encryption key is expected')
@@ -244,6 +263,15 @@ module.exports = class Autobee extends ReadyResource {
 
     this._registerWakeup()
 
+    if (this.wakeupCapability) {
+      if (this.bootstrap !== this.local) {
+        await this.bootstrap.setGroup(this.wakeupCapability.discoveryKey)
+      }
+
+      this._notifyHandler = this.store.notifyGroup(this.wakeupCapability.discoveryKey)
+      this._notifyHandler.on('update', this._bumpSoonBound)
+    }
+
     const system = result.system || EMPTY_HEAD
 
     await this.system.boot(system)
@@ -256,10 +284,14 @@ module.exports = class Autobee extends ReadyResource {
     this.bee.move(view)
 
     await this.writers.updateLocalState()
+
+    this._bootGuard.exit()
+
+    return this._bootGuard.ready()
   }
 
   async _bootAll() {
-    await this._bootingState
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
 
     for await (const node of this.system.list()) {
       await this.writers.add(node.key)
@@ -272,7 +304,10 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _bump() {
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
+
     await this._flushWakeup()
+
     this.bumping++
 
     if (!this._draining) {
@@ -327,18 +362,30 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _drain() {
-    if (this._updateLocalCore !== null) {
-      await this._rotateLocalWriter(this._updateLocalCore)
+    if (this.fastForwardBoot) {
+      this._initFastForward(this.fastForwardBoot)
+      this.fastForwardBoot = null
     }
 
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
+    // Anything expecting work to be done during bumpSoon should do it here
     while (!this._interrupting && this.bumping > 0) {
-      if (this._interrupting) return
+      if (this._interrupting) break
+
+      // Ensure we catch updates during the drain (i.e. setLocal will bump)
+      if (this._updateLocalCore !== null) {
+        await this._rotateLocalWriter(this._updateLocalCore)
+      }
 
       try {
         while (!this._interrupting) {
+          if (this.fastForwardTo !== null) {
+            await this._applyFastForward()
+            break // revaluate conditions...
+          }
+
           if (!(await this._bumpPendingWriters())) break
           this._needsUpdate = true
         }
@@ -361,6 +408,16 @@ module.exports = class Autobee extends ReadyResource {
   async _flushWakeup() {
     const hints = this._wakeup.flush()
 
+    if (this._notifyHandler) {
+      for await (const key of this._notifyHandler.updates({ since: this.previousDrain })) {
+        const hex = b4a.toString(key, 'hex')
+        hints.set(hex, -1)
+      }
+    }
+
+    this.previousDrain = Date.now()
+    this.queueWakeupFastForward(hints).catch(noop)
+
     for (const [hex, length] of hints) {
       const key = b4a.from(hex, 'hex')
       if (this.writers.has(hex)) continue
@@ -372,6 +429,25 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
+  _initFastForward({ key, length }) {
+    this.moveTo({ key, length })
+  }
+
+  async _getOplog(key, length) {
+    const core = this.openCore(key)
+    await core.ready()
+
+    const target = length >= 0 ? length : core.length
+    if (target === 0) return null
+
+    const buf = await core.get(target - 1)
+    await core.close()
+
+    if (buf === null) return null
+
+    return encoding.decodeOplog(buf)
+  }
+
   _update(changes) {
     this._needsUpdate = false
     this.bee.update(this._workingBee.root)
@@ -407,7 +483,7 @@ module.exports = class Autobee extends ReadyResource {
 
     let runs = 0
     while (!this._interrupting && this.appending && runs++ < 16) await this.update()
-    await this.bumpSoon()
+    this.bumpSoon()
   }
 
   async _rotateLocalWriter(newLocal) {
@@ -573,6 +649,10 @@ module.exports = class Autobee extends ReadyResource {
       this._workingBee.move(t.view)
     }
 
+    return this._processApplyBatch(t)
+  }
+
+  async _processApplyBatch(t) {
     // first writer is always added with full permissions
     if (this.system.isGenesis()) {
       await this._host.addWriter(t.tip[0][0].key)
@@ -609,9 +689,20 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   _storeBoot() {
+    const proms = []
+    proms.push(
+      this.local.setUserData(
+        'autobee/previous-drain',
+        encoding.encodePreviousDrain(this.previousDrain)
+      )
+    )
+
     const boot = this.system.bootRecord()
-    if (!boot) return
-    return this.local.setUserData('autobee/head', encoding.encodeBootRecord(boot))
+    if (boot) {
+      proms.push(this.local.setUserData('autobee/head', encoding.encodeBootRecord(boot)))
+    }
+
+    return Promise.all(proms)
   }
 
   static decodeValue(buf, opts) {
@@ -623,7 +714,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async wakeup({ key, length }) {
-    await this._bootingState
+    if (!this._bootGuard.opened) await this._bootGuard.ready()
     await this.writers.wakeup(key, length)
     await this._bump()
   }
@@ -655,8 +746,119 @@ module.exports = class Autobee extends ReadyResource {
     await this.writers.flushLocal(this._workingBee.head())
   }
 
-  async replay() {
-    return topo.replay(this)
+  moveTo(head, tip) {
+    if (this.fastForwardTo !== null || this.fastForwarding !== null) return null
+    return this._runFastForward(new FastForward(this, head, tip)).catch(noop)
+  }
+
+  async queueWakeupFastForward(hints) {
+    if (!this._handlers.onwakeup) return false
+    if (!hints.size || this.fastForwarding || this.fastForwardTo || this.fastForwardBoot) {
+      return false
+    }
+
+    const promises = []
+    for (const [hex, length] of hints) {
+      if (length === 0) continue
+      const key = b4a.from(hex, 'hex')
+      promises.push(this._getOplog(key, length))
+    }
+
+    const ops = await Promise.all(promises)
+    if (this.fastForwarding || this.fastForwardTo) return false
+
+    let best = null
+    let bestFlushes = -1
+
+    for (const msg of ops) {
+      if (msg === null) continue
+
+      if (msg.views && msg.views.flushes > bestFlushes) {
+        bestFlushes = msg.views.flushes
+        best = msg.views
+      }
+    }
+
+    if (best === null || bestFlushes - this.system.flushes < MIN_FF_GAP) return false
+
+    const view = this.bee.checkout(best.view)
+
+    let trusted = null
+    try {
+      trusted = await this._handlers.onwakeup(view, this)
+      if (!trusted || this.fastForwarding || this.fastForwardTo) return false
+    } finally {
+      view.close()
+    }
+
+    const oplog = await this._getOplog(trusted.key, trusted.length)
+
+    return this.moveTo(oplog.views.system, {
+      system: best.system,
+      verified: {
+        node: trusted,
+        flushes: oplog.views.flushes
+      }
+    })
+  }
+
+  async _runFastForward(ff) {
+    this.fastForwarding = ff
+
+    const result = await ff.upgrade()
+    await ff.close()
+
+    if (this.fastForwarding === ff) this.fastForwarding = null
+
+    if (!result) return null
+
+    this.fastForwardTo = result
+    this.fastForward = rrp()
+
+    this.bumpSoon()
+
+    return this.fastForward.promise
+  }
+
+  async _applyFastForward() {
+    const changes = this._hasUpdate ? new UpdateChanges(this) : null
+    if (changes) changes.track(this._applyState)
+
+    const { head, tip } = this.fastForwardTo
+
+    const from = this.system.bee.head()
+    const to = head
+
+    this.system.bee.move(head)
+    await this.system.reset()
+
+    this.bee.move(this.system.view)
+    this._workingBee.move(this.system.view)
+
+    this.fastForwardTo = null
+
+    await this.writers.refresh()
+
+    if (changes) {
+      changes.finalise()
+      await this._handlers.update(this.view, changes)
+    }
+
+    this.emit('move-to', to, from)
+    this.fastForward.resolve({ to, from })
+
+    // tip is null when handlers.fastForward set
+    if (!tip) return
+
+    return this._reapply(tip)
+  }
+
+  async _reapply({ system, verified }) {
+    const sys = this.system.bee.checkout(system)
+    const t = await topo.rollback(this, sys, verified)
+    await sys.close()
+
+    return this._processApplyBatch(t)
   }
 }
 

package/lib/boot.js

@@ -11,7 +11,8 @@ module.exports = async function boot(
     bootstrap: null,
     encryptionKey: null,
     system: null,
-    flushes: null
+    flushes: null,
+    previousDrain: 0
   }
 
   const manifest = keyPair
@@ -91,15 +92,15 @@ module.exports = async function boot(
     await result.local.setUserData('referrer', result.key)
   }
 
-  const [systemHead, encryptionKeyBuffer] = await Promise.all([
+  const [systemHead, encryptionKeyBuffer, prevDrainBuffer] = await Promise.all([
     result.local.getUserData('autobee/head'),
-    result.local.getUserData('autobee/encryption')
+    result.local.getUserData('autobee/encryption'),
+    result.local.getUserData('autobee/previous-drain')
   ])
 
   if (systemHead) result.system = encoding.decodeBootRecord(systemHead)
-  if (encryptionKeyBuffer) {
-    result.encryptionKey = encryptionKeyBuffer
-  }
+  if (encryptionKeyBuffer) result.encryptionKey = encryptionKeyBuffer
+  if (prevDrainBuffer) result.previousDrain = encoding.decodePreviousDrain(prevDrainBuffer)
 
   if (!result.encryptionKey && (encryptionKey || encrypt)) {
     if (!encryptionKey) {

package/lib/encoding.js

@@ -18,6 +18,8 @@ exports.Oplog = Oplog
 exports.ManifestData = ManifestData
 exports.encodeBootRecord = encodeBootRecord
 exports.decodeBootRecord = decodeBootRecord
+exports.encodePreviousDrain = encodePreviousDrain
+exports.decodePreviousDrain = decodePreviousDrain
 exports.encodeSystemWriter = encodeSystemWriter
 exports.decodeSystemWriter = decodeSystemWriter
 
@@ -29,6 +31,14 @@ function decodeBootRecord(value) {
   return c.decode(Link, value)
 }
 
+function encodePreviousDrain(d) {
+  return c.encode(c.uint, d)
+}
+
+function decodePreviousDrain(value) {
+  return c.decode(c.uint, value)
+}
+
 function encodeSystemWriter(m) {
   return c.encode(SystemWriter, m)
 }

package/lib/fast-forward.js

@@ -0,0 +1,71 @@
+const safetyCatch = require('safety-catch')
+
+const System = require('./system.js')
+
+const DEFAULT_OP_TIMEOUT = 5_000
+
+module.exports = class FastForward {
+  constructor(auto, head, tip, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
+    this.auto = auto
+    this.system = new System(auto.store.namespace('fast-forward'), null, {
+      getEncryptionProvider: this.auto._getEncryptionProviderBound,
+      encrypted: this.encrypted
+    })
+
+    this.head = head
+    this.timeout = timeout
+    this.tip = tip || null
+    this.destroyed = false
+    this.upgrading = null
+    this.failed = false
+    this.cores = []
+  }
+
+  async upgrade() {
+    try {
+      if (!this.upgrading) this.upgrading = this._upgrade()
+
+      if (!(await this.upgrading)) return null
+
+      return {
+        head: this.head,
+        tip: this.tip
+      }
+    } catch (err) {
+      safetyCatch(err)
+      this.failed = true
+      return null
+    } finally {
+      await this.close()
+    }
+  }
+
+  async _upgrade() {
+    await this.system.boot(this.head)
+
+    const promises = []
+
+    // ensure local key is locally available always
+    promises.push(this.system.get(this.auto.local.key, { timeout: this.timeout }))
+
+    const view = this.auto.store.get({ key: this.system.view.key, active: true })
+    this.cores.push(view)
+
+    promises.push(view.get(this.system.view.length - 1, { timeout: this.timeout }))
+
+    for (const head of this.system.heads) {
+      promises.push(this.system.get(head.key, { timeout: this.timeout }))
+    }
+
+    await Promise.all(promises)
+    if (this.destroyed) return false
+
+    return true
+  }
+
+  async close() {
+    this.destroyed = true
+    if (this.system) await this.system.close()
+    for (const core of this.cores) await core.close()
+  }
+}

package/lib/topo.js

@@ -5,8 +5,8 @@ const asserts = require('./asserts.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 
-exports.replay = replay
 exports.sort = sort
+exports.rollback = rollback
 
 exports.isLinking = isLinking
 exports.isLinkingAll = isLinkingAll
@@ -29,17 +29,29 @@ class Clock {
   }
 }
 
-async function replay(auto) {
-  const ch = auto.system.bee.createChangesStream()
-  const replay = []
+async function rollback(auto, system, verified) {
+  const tip = []
+  const ch = system.createChangesStream()
 
   for await (const data of ch) {
     const { ref, info } = getOplog(data)
-    replay.push(ref)
+    if (sameNode(ref.oplog, verified.node)) {
+      asserts.assert(info.flushes === verified.flushes, 'Expect tip to be continuous')
+      break
+    }
+
+    tip.push(ref)
   }
 
-  const inflated = await inflate(auto, replay.reverse())
-  return inflated.flatMap((b) => b.batch)
+  const inflated = await inflate(auto, tip)
+
+  // @todo: verify sort
+
+  return {
+    undo: null,
+    view: null,
+    tip: inflated.map(getBatch)
+  }
 }
 
 async function sort(auto, batch) {
@@ -269,3 +281,11 @@ function isLinkingAll(node, heads) {
 
   return true
 }
+
+function sameNode(a, b) {
+  return b4a.equals(a.key, b.key) && a.length === b.length
+}
+
+function getBatch(b) {
+  return b.batch
+}

package/lib/writers.js

@@ -37,7 +37,7 @@ class ActiveWriters {
   }
 
   bump() {
-    this.auto._bump()
+    this.auto.bumpSoon()
   }
 
   async wakeup(key, length) {
@@ -66,6 +66,7 @@ class ActiveWriters {
 
     this.localWriter = new Writer(this, this.auto.system, true, core, b4a.toString(core.key, 'hex'))
     this.localWriter.addPending()
+    this.auto.emit('writer', this.localWriter)
   }
 
   clearLocal() {
@@ -76,6 +77,20 @@ class ActiveWriters {
     return this.localWriter.flush(head)
   }
 
+  async refresh() {
+    if (this.localWriter && !this.localWriter.closed) {
+      await this.updateLocalState()
+    }
+
+    for (const writer of this.active.values()) {
+      await writer.update()
+
+      if (writer !== this.localWriter && !writer.isIndexer) {
+        await writer.detachAndClose()
+      }
+    }
+  }
+
   async updateLocalState() {
     const w = this.localWriter
     const bootstrapping = w.core.length === 0 && b4a.equals(this.auto.key, w.core.key)
@@ -96,7 +111,8 @@ class ActiveWriters {
   }
 
   async add(key) {
-    if (b4a.equals(key, this.auto.local.key)) await this.updateLocalState()
+    const local = b4a.equals(key, this.auto.local.key)
+    if (local) await this.updateLocalState()
 
     const id = b4a.toString(key, 'hex')
     let w = this.active.get(id)
@@ -109,8 +125,10 @@ class ActiveWriters {
 
     const core = this.auto.store.get({
       key,
-      encryption
+      encryption,
+      group: local || !this.auto.wakeupCapability ? null : this.auto.wakeupCapability.discoveryKey
     })
+
     await core.ready()
     await core.setUserData('referrer', this.auto.key)
 
@@ -239,13 +257,14 @@ class Writer {
     if (this.isAttached) return
     this.isAttached = true
     this.writers.active.set(this.id, this)
+    this.writers.auto.emit('writer', this)
     this.addPending()
   }
 
   detach() {
     if (!this.isAttached) return
     this.isAttached = false
-    this.download.destroy()
+    if (this.download) this.download.destroy()
     this.download = null
     this.writers.active.delete(this.id)
     this.removePending()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -28,15 +28,17 @@
     "b4a": "^1.8.0",
     "compact-encoding": "^2.18.0",
     "hyperbee2": "^2.8.0",
-    "hypercore": "^11.28.1",
+    "hypercore": "^11.33.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.20.1",
+    "ready-guard": "^1.0.0",
     "ready-resource": "^1.2.0",
+    "resolve-reject-promise": "^1.1.0",
     "safety-catch": "^1.0.3"
   },
   "devDependencies": {
     "brittle": "^3.19.1",
-    "corestore": "^7.7.0",
+    "corestore": "^7.10.0",
     "hypercore-crypto": "^3.6.1",
     "lunte": "^1.6.0",
     "prettier": "^3.8.1",