autobee 2.0.0-rc.17 → 2.0.0-rc.19

package/index.js

@@ -235,6 +235,8 @@ module.exports = class Autobee extends ReadyResource {
       await this._draining
     }
 
+    if (this._updating) await this._updating
+
     // let in-flight writer adds finish
     try {
       await this._bootingAll
@@ -263,25 +265,6 @@ module.exports = class Autobee extends ReadyResource {
     if (this.fastForwarding) this.fastForwarding.clearRequests(closingError)
   }
 
-  async changesFrom({ system } = {}) {
-    const info = await this.system.getInfo()
-
-    const current = {
-      flushes: info ? info.flushes : 0,
-      system: this.system.bee.head(),
-      view: info ? info.view : EMPTY_HEAD
-    }
-
-    // no common ancestor (null) -> flushes -1 never matches, forcing a full diff
-    const shared = (await this.system.commonAncestor(system || EMPTY_HEAD)) || {
-      flushes: -1,
-      system: EMPTY_HEAD,
-      view: EMPTY_HEAD
-    }
-
-    return UpdateChanges.from(shared, current)
-  }
-
   replicate(...args) {
     const stream = this.store.replicate(...args)
     this._wakeup.addStream(stream)
@@ -529,9 +512,6 @@ module.exports = class Autobee extends ReadyResource {
       else if (head) await this._bootFromHead(head, bootCondition)
     }
 
-    // tracked across drain iteration
-    this.system.shared = null
-
     this._ackRequired = false
 
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
@@ -566,6 +546,8 @@ module.exports = class Autobee extends ReadyResource {
         }
 
         await this._flushLocal()
+
+        if (!this._interrupting) await this.writers.refresh()
       } finally {
         if (this.bumping === 1) this.bumping = 0
         else this.bumping = 1
@@ -575,16 +557,15 @@ module.exports = class Autobee extends ReadyResource {
     this._draining = null
     if (this._interrupting) return
 
-    if (this._needsUpdate) {
-      const updating = rrp()
-      this._updating = updating.promise
+    const updating = rrp()
+    this._updating = updating.promise
 
-      try {
-        await this._update(changes)
-      } finally {
-        this._updating = null
-        updating.resolve()
-      }
+    try {
+      if (this._needsUpdate) await this._update(changes)
+      await this._storeBoot()
+    } finally {
+      this._updating = null
+      updating.resolve()
     }
   }
 
@@ -617,18 +598,28 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _flushWakeup() {
+  // wakeup hints keep arriving asynchronously
+  async _applyWakeupHints() {
     const hints = this._wakeup.flush()
-    if (!hints.size) return
+    if (!hints.size) return hints
 
     this.previousDrain = Date.now()
 
     for (const [hex, length] of hints) {
       const key = b4a.from(hex, 'hex')
-      if (this.writers.has(hex)) continue
+      // wakeup() itself no-ops the add for an already-active writer, but still
+      // needs to run so it can hint() the announced length - skipping it here
+      // for active writers left gc with no hint to protect them
       await this.writers.wakeup(key, length)
     }
 
+    return hints
+  }
+
+  async _flushWakeup() {
+    const hints = await this._applyWakeupHints()
+    if (!hints.size) return
+
     // a scheduled fast-forward is applied by the drain before we look again
     if (this.fastForwardTo !== null || this.fastForwarding !== null) return
     if (this._interrupting || this.closing || this.bootFrom) return
@@ -918,41 +909,33 @@ module.exports = class Autobee extends ReadyResource {
       this._catchupMigratedNodes = null
     }
 
-    let updated = false
-
-    const pending = this.writers.pending.slice()
+    // apply the best next node to keep the prefix stable
+    const next = await this.writers.nextPendingNode()
+    if (next === null) return false
 
-    for (let i = pending.length - 1; i >= 0; i--) {
-      const w = pending[i]
+    const { writer: w, batch } = next
 
-      const batch = await w.next()
-      if (batch === null) continue
-
-      if (w.isAdded || (w.isRemoved && w.hasReferrals())) {
-        await this._processBatch(batch)
-        w.notify(batch)
-        updated = true
-        continue
-      }
+    if (w.isAdded || (w.isRemoved && w.hasReferrals())) {
+      await this._processBatch(batch)
+      w.notify(batch)
+      return true
+    }
 
-      if (this.optimistic && !w.isRemoved && batch[0].optimistic) {
-        if (!(await this._optimisticBatch(batch))) {
-          w.removePending()
-          continue
-        }
-        w.notify(batch)
-        updated = true
-        continue
+    if (this.optimistic && !w.isRemoved && batch[0].optimistic) {
+      if (!(await this._optimisticBatch(batch))) {
+        w.removePending()
+        return true
       }
+      w.notify(batch)
+      return true
     }
 
-    return updated
+    return true
   }
 
   async _optimisticBatch(batch) {
     const rollbackSystem = this.system.bee.head()
     const rollbackView = this._workingBee.head()
-    const rollbackShared = this.system.shared
     const rollbackAttestations = this.writers.attestations.length
 
     const t = await this.prepareBatch(batch)
@@ -975,7 +958,6 @@ module.exports = class Autobee extends ReadyResource {
         this._workingBee.move(rollbackView)
         this.system.bee.move(rollbackSystem)
         await this.system.reset()
-        this.system.shared = rollbackShared
         // don't attest grants that were just undone
         this.writers.attestations.length = rollbackAttestations
         return false
@@ -1064,8 +1046,6 @@ module.exports = class Autobee extends ReadyResource {
 
     this.writers.attest(witnessed)
 
-    await this._storeBoot()
-
     for (const { key, added, isAnchor } of changed) {
       if (isAnchor) {
         // no Writer/ActiveWriters tracking, but still wake anything linked to it
@@ -1327,12 +1307,6 @@ module.exports = class Autobee extends ReadyResource {
     this.system.bee.move(head)
     await this.system.reset()
 
-    this.system.shared = this.fastForwardTo.shared || {
-      flushes: -1,
-      view: EMPTY_HEAD,
-      system: EMPTY_HEAD
-    }
-
     // migrate is set when fast-forwarding from a legacy head
     if (migrate) {
       const view = (await this._handlers.migrate(migrate)) || EMPTY_HEAD
@@ -1345,14 +1319,16 @@ module.exports = class Autobee extends ReadyResource {
     }
 
     this.fastForwardTo = null
+
+    // process any wakeup while fast-forward itself was in flight
+    await this._applyWakeupHints()
     await this.writers.refresh()
 
     // we moved, so ask our peers to tell us their heads again
     this._requestWakeup()
 
-    await this._storeBoot()
-
     await this._update(changes)
+    await this._storeBoot()
 
     this.stats.fastForwards++
     this.emit('move-to', to, from)

package/lib/fast-forward.js

@@ -261,23 +261,12 @@ module.exports = class FastForward {
     await Promise.all(promises)
     if (this.destroyed) return null
 
-    const shared = await this._commonAncestor()
     if (this.destroyed) return null
 
     return {
       head: this.head,
       tip: this.tip,
-      migrate: null,
-      shared
-    }
-  }
-
-  async _commonAncestor() {
-    try {
-      return await this.auto.system.commonAncestor(this.head)
-    } catch (err) {
-      safetyCatch(err)
-      return null
+      migrate: null
     }
   }
 

package/lib/system.js

@@ -46,7 +46,6 @@ module.exports = class Systembee {
     this.indexers = null // legacy
     this.updates = new Map()
     this.writers = new Map()
-    this.shared = null
     this.encrypted = opts.encrypted === true
   }
 
@@ -115,60 +114,6 @@ module.exports = class Systembee {
     }
   }
 
-  async commonAncestor(head) {
-    const snap = this.bee.checkout(head)
-
-    const ca = infoIterator(this.bee)
-    const cb = infoIterator(snap)
-
-    let l = await next(ca)
-    let r = await next(cb)
-
-    while (l && r) {
-      if (l.legacy && r.legacy) {
-        // legacy nodes just compare head
-        if (sameHead(l.node.head, r.node.head)) {
-          return {
-            flushes: l.info.flushes,
-            system: l.node.head,
-            view: l.info.view || EMPTY_HEAD
-          }
-        }
-
-        // different legacy cores never converge - genuinely unrelated history
-        if (!b4a.equals(l.node.head.key, r.node.head.key)) return null
-
-        if (l.node.head.length >= r.node.head.length) l = await next(ca)
-        else r = await next(cb)
-        continue
-      }
-
-      if (l.legacy !== r.legacy) {
-        // current-format history sits above the legacy band: descend it
-        if (!l.legacy) l = await next(ca)
-        else r = await next(cb)
-        continue
-      }
-
-      if (l.info.flushes === r.info.flushes && sameHead(l.info.view, r.info.view)) {
-        return {
-          flushes: l.info.flushes,
-          system: l.node.head,
-          view: l.info.view || EMPTY_HEAD
-        }
-      }
-
-      const lf = l.info.flushes
-      const rf = r.info.flushes
-
-      if (lf >= rf) l = await next(ca)
-      if (rf >= lf) r = await next(cb)
-    }
-
-    // no common ancestor
-    return null
-  }
-
   async reset({ timeout } = {}) {
     const info = await this.getInfo({ timeout })
 
@@ -178,7 +123,6 @@ module.exports = class Systembee {
     this.timestamp = info ? info.timestamp : 0
     this.flushes = info ? info.flushes : 0
     this.indexers = info ? info.indexers : null
-    this.shared = null
     this.updates.clear()
     this.writers.clear()
 
@@ -200,16 +144,8 @@ module.exports = class Systembee {
   async undo(head) {
     this.bee.move(head)
 
-    const shared = this.shared
     await this.reset()
 
-    // deepest rewind since last mark - everything at or below it is untouched
-    if (shared === null || this.flushes < shared.flushes) {
-      this.shared = { flushes: this.flushes, view: this.view || EMPTY_HEAD, system: head }
-    } else if (shared) {
-      this.shared = shared
-    }
-
     return this.view || EMPTY_HEAD
   }
 
@@ -514,41 +450,3 @@ function getInfo(data) {
 
   return null
 }
-
-function sameHead(a, b) {
-  if (!a || !b || a.length !== b.length) return false
-  if (!a.key || !b.key) return a.key === b.key
-  return b4a.equals(a.key, b.key)
-}
-
-async function* infoIterator(bee) {
-  try {
-    for await (const node of bee.createChangesStream()) {
-      const legacy = topo.isLegacyNode(node)
-      const info = legacy ? getLegacyInfo(node) : getInfo(node)
-      if (info) yield { node, info, legacy }
-    }
-  } catch {
-    // fall through
-  }
-}
-
-async function next(it) {
-  const res = await it.next()
-  return res.done ? null : res.value
-}
-
-function getLegacyInfo(data) {
-  for (const { keys } of data.batch) {
-    for (const k of keys) {
-      if (b4a.equals(k.key, INFO_LEGACY_KEY)) {
-        try {
-          return encoding.decodeSystemInfo(k.value)
-        } catch {
-          return null
-        }
-      }
-    }
-  }
-  return null
-}

package/lib/topo.js

@@ -6,6 +6,8 @@ const asserts = require('./asserts.js')
 const EMPTY_HEAD = { length: 0, key: null }
 const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 
+const DEFAULT_OP_TIMEOUT = 5000
+
 exports.sort = sort
 exports.rollback = rollback
 exports.replay = replay
@@ -229,6 +231,8 @@ function cmp(a, b) {
 }
 
 async function inflate(auto, reorder, opts = null) {
+  const readOpts = opts === null ? { timeout: DEFAULT_OP_TIMEOUT } : opts
+
   const promises = []
   const sessions = new Map()
 
@@ -257,7 +261,7 @@ async function inflate(auto, reorder, opts = null) {
       sessions.set(id, core)
     }
 
-    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i, opts))
+    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i, readOpts))
   }
 
   try {

package/lib/updates.js

@@ -1,5 +1,3 @@
-const b4a = require('b4a')
-
 module.exports = class UpdateChanges {
   constructor(auto) {
     this.auto = auto
@@ -21,57 +19,22 @@ module.exports = class UpdateChanges {
         flushes: this.auto.system.flushes,
         from: this.tracking[k].head(),
         to: null,
-        shared: null
+        incremental: false
       })
     }
   }
 
   finalise() {
-    const floor = this.auto.system.shared
-
     // flush count at `to` - safe to read live, state is frozen for the hook
     const flushes = this.auto.system.flushes
 
     for (const k in this.tracking) {
       const tracked = this.byName.get(k)
-      const startFlushes = tracked.flushes
 
       tracked.to = this.tracking[k].head()
       tracked.flushes = flushes
-
-      if (floor && floor.flushes < startFlushes) {
-        tracked.shared = k === 'view' ? floor.view : floor.system
-      } else {
-        tracked.shared = tracked.from
-      }
     }
 
     return this.byName
   }
-
-  static from(shared, current) {
-    if (
-      shared.flushes === current.flushes &&
-      shared.system.length === current.system.length &&
-      (shared.system.key === current.system.key ||
-        (shared.system.key &&
-          current.system.key &&
-          b4a.equals(shared.system.key, current.system.key)))
-    ) {
-      return null
-    }
-
-    const byName = new Map()
-    const flushes = current.flushes
-
-    byName.set('view', { from: shared.view, shared: shared.view, to: current.view, flushes })
-    byName.set('system', {
-      from: shared.system,
-      shared: shared.system,
-      to: current.system,
-      flushes
-    })
-
-    return byName
-  }
 }

package/lib/writers.js

@@ -1,7 +1,7 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
 const Triggers = require('./triggers.js')
-const { signAttestation } = require('./witness.js')
+const { signAttestation, resolveWeight } = require('./witness.js')
 const { WriterEncryption } = require('autobee-encryption')
 
 const WRITER_PREFETCH = 10
@@ -25,6 +25,8 @@ class ActiveWriters {
     this.witnesses = []
     this.attestations = []
 
+    this._pendingBuffer = new Map()
+
     this.localWriter.attach()
 
     this._bootstrap()
@@ -62,13 +64,15 @@ class ActiveWriters {
       const info = await this.auto.system.get(key, { unflushed: true })
       if (info && info.isAnchor) return
       w = await this.add(key)
+      if (!w) return
     }
 
+    // remember the announced length so gc doesn't drop a writer we haven't caught up on
+    w.hint(length)
+
     // prefetch the oplog head for the fast-forward that runs right after us
     if (w && length > 0) w.core.get(length - 1).catch(noop)
 
-    // TODO: signal that we've seen length on w for gc
-
     this.bump()
   }
 
@@ -113,6 +117,9 @@ class ActiveWriters {
   }
 
   async refresh() {
+    // peeked batches were read against the pre-reboot system, drop them
+    this._pendingBuffer.clear()
+
     if (this.localWriter && !this.localWriter.isClosed) {
       await this.updateLocalState()
     }
@@ -127,6 +134,33 @@ class ActiveWriters {
     await this.localWriter.update()
   }
 
+  async nextPendingNode() {
+    for (const w of this.pending.slice()) {
+      if (this._pendingBuffer.has(w) || w.isFrozen) continue
+
+      const batch = await w.next()
+      if (batch !== null) this._pendingBuffer.set(w, batch)
+    }
+
+    if (this._pendingBuffer.size === 0) return null
+
+    let writer = null
+    let best = null
+
+    for (const [w, batch] of this._pendingBuffer) {
+      const head = batch[0]
+      // -1 marks a node next() has not resolved a sort weight for yet
+      if (head.weight < 0) head.weight = await resolveWeight(this.auto, head)
+      if (best === null || sortsBefore(head, best[0])) {
+        writer = w
+        best = batch
+      }
+    }
+
+    this._pendingBuffer.delete(writer)
+    return { writer, batch: best }
+  }
+
   _updateLocalState() {
     const w = this.localWriter
 
@@ -215,6 +249,7 @@ class ActiveWriters {
     const id = b4a.toString(bootstrap.key, 'hex')
     let w = this.active.get(id)
     if (w) {
+      w.isBootstrap = true
       w.couple() // couple if we weren't ready first time
       return
     }
@@ -223,6 +258,7 @@ class ActiveWriters {
     session.setEncryption(this.auto._getEncryptionProvider())
 
     w = new Writer(this, this.auto.system, false, session, id)
+    w.isBootstrap = true // the root of the autobase, never gc'd
     w.attach()
     w.couple()
   }
@@ -253,6 +289,7 @@ class Writer {
     this.appendLength = this.core.length + 1
     this.waiting = null
     this.download = null
+    this.hintedLength = 0
 
     this.isPending = false
     this.isClosed = false
@@ -261,6 +298,7 @@ class Writer {
     this.isRemoved = false
     this.isCoupled = false
     this.isFrozen = false
+    this.isBootstrap = false
 
     this._onchangeBound = this._onchange.bind(this)
 
@@ -308,6 +346,8 @@ class Writer {
     if (!this.isPending) return
     this.isPending = false
 
+    this.writers._pendingBuffer.delete(this)
+
     const head = this.writers.pending.pop()
     if (head !== this) {
       this.writers.pending[this.index] = head
@@ -315,6 +355,11 @@ class Writer {
     }
   }
 
+  // a peer announced this length for us, so we're not caught up until we've seen it
+  hint(length) {
+    if (length > this.hintedLength) this.hintedLength = length
+  }
+
   _onchange() {
     if (this.waiting === null) this.bump()
   }
@@ -352,10 +397,10 @@ class Writer {
 
     const length = await this.update()
 
-    // don't drop any local nodes left in buffer
-    if (length < this.core.length || this.appending) {
+    // don't drop any local nodes left in buffer, or work a peer told us about
+    if (length < this.core.length || length < this.hintedLength || this.appending) {
       this.addPending()
-    } else if (this !== this.writers.localWriter && !this.isIndexer) {
+    } else if (this !== this.writers.localWriter && !this.isIndexer && !this.isBootstrap) {
       await this.detachAndClose()
     }
   }
@@ -375,6 +420,7 @@ class Writer {
 
   detachAndClose() {
     this.detach()
+    this.decouple()
     return this.close()
   }
 
@@ -690,6 +736,17 @@ class Writer {
   }
 }
 
+// mirrors topo.cmp - kept local because topo.js requires this module
+function sortsBefore(a, b) {
+  if (a.weight !== b.weight) return a.weight > b.weight
+  if (a.timestamp !== b.timestamp) return a.timestamp < b.timestamp
+
+  const c = b4a.compare(a.key, b.key)
+  if (c !== 0) return c < 0
+
+  return a.length < b.length
+}
+
 exports.ActiveWriters = ActiveWriters
 exports.Writer = Writer
 exports.createNode = createNode

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.17",
+  "version": "2.0.0-rc.19",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",