autobee 2.7.3 → 2.8.0

package/index.js

@@ -69,7 +69,15 @@ module.exports = class Autobee extends ReadyResource {
     this.id = null
     this.bootstrap = null
     this._handlers = handlers
-    this.stats = { undos: 0, fastForwards: 0, drains: 0, applies: 0, appends: 0 }
+    this.stats = {
+      undos: 0,
+      fastForwards: 0,
+      drains: 0,
+      applies: 0,
+      appends: 0,
+      writersOpened: 0,
+      writersClosed: 0
+    }
 
     const systemStore = this.store.session()
     this.system = new System(this, systemStore, {
@@ -205,9 +213,13 @@ module.exports = class Autobee extends ReadyResource {
     if (this._acking) this.bumpSoon()
   }
 
+  isLocalTrusted() {
+    return this.trusted.isTrusted(this.local.key, this._workingView.view)
+  }
+
   async _updateAcking() {
     if (this._interrupting) return
-    this.setAcking(await this.trusted.isTrusted(this.local.key, this._workingView.view))
+    this.setAcking(await this.isLocalTrusted())
   }
 
   // network free: only a migration needs peers, so ready() awaits the full
@@ -1032,10 +1044,9 @@ module.exports = class Autobee extends ReadyResource {
     return anchor
   }
 
-  // see CatchupOrder - sort order instead of legacy INFO order, so the catchup
-  // does not rebase on itself
+  // apply in linearizer order so the catchup never undoes. weights are fixed,
+  // legacy nodes carry no witness
   async _bumpMigratedWriters() {
-    const order = new migrations.CatchupOrder(this._catchupMigratedNodes)
     const opened = new Set()
     let updated = false
 
@@ -1046,16 +1057,14 @@ module.exports = class Autobee extends ReadyResource {
     }
 
     try {
-      while (order.size) {
-        const candidates = order.candidates()
-        // recomputed per step - prepareBatch does the same right after
-        for (const batch of candidates) batch[0].weight = await resolveWeight(this, batch[0])
+      for (const batch of this._catchupMigratedNodes) {
+        const weight = await resolveWeight(this, batch[0])
+        for (const node of batch) node.weight = weight
+      }
 
-        const best = order.pick(candidates)
-        await this._processBatch(best)
+      for (const batch of topo.linearize(this._catchupMigratedNodes)) {
+        await this._processBatch(batch)
         updated = true
-
-        order.shift(best)
       }
     } finally {
       for (const core of opened) await core.close()

package/lib/migrations.js

@@ -6,7 +6,6 @@ const { decodeBlock } = require('hyperbee2/lib/encoding.js')
 
 const encoding = require('./encoding.js')
 const { assert, bail } = require('./asserts.js')
-const { sortsBefore } = require('./writers.js')
 
 const { getEncoding } = require('../encoding/spec/autobee')
 
@@ -18,77 +17,13 @@ const INDEX_VERSION = 1
 const INFO = b4a.from([0x00, 0x00, 0x69, 0x6e, 0x66, 0x6f])
 const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
 
-// hands out the catchup in sort order rather than legacy INFO order, so it
-// does not rebase on itself. only heads whose links are satisfied are
-// candidates, and the best of those by sortsBefore goes first. the caller
-// resolves weights on the candidates before picking
-class CatchupOrder {
-  constructor(batches) {
-    this.queues = new Map()
-    this.applied = new Map()
-
-    for (const batch of batches) {
-      const id = b4a.toString(batch[0].key, 'hex')
-      const q = this.queues.get(id)
-      if (q) q.push(batch)
-      else this.queues.set(id, [batch])
-    }
-
-    for (const [id, q] of this.queues) this.applied.set(id, q[0][0].length - 1)
-  }
-
-  get size() {
-    return this.queues.size
-  }
-
-  candidates() {
-    const batches = []
-
-    for (const q of this.queues.values()) {
-      if (this._isCandidate(q[0][0])) batches.push(q[0])
-    }
-
-    assert(batches.length > 0 || this.queues.size === 0, 'Migration catchup has unresolvable links')
-
-    return batches
-  }
-
-  pick(candidates) {
-    let best = null
-    for (const batch of candidates) {
-      if (best === null || sortsBefore(batch[0], best[0])) best = batch
-    }
-    return best
-  }
-
-  shift(batch) {
-    const id = b4a.toString(batch[0].key, 'hex')
-    const q = this.queues.get(id)
-
-    assert(q && q[0] === batch, 'Catchup batch shifted out of order')
-
-    q.shift()
-    this.applied.set(id, batch[batch.length - 1].length)
-    if (!q.length) this.queues.delete(id)
-  }
-
-  _isCandidate(node) {
-    for (const l of node.links) {
-      const length = this.applied.get(b4a.toString(l.key, 'hex'))
-      if (length !== undefined && length < l.length) return false
-    }
-    return true
-  }
-}
-
 module.exports = {
   getCoreManifest,
   getViewMap,
   checkAutobaseMigration,
   readLegacySystemInfo,
   inflateLegacyOplog,
-  coreLength,
-  CatchupOrder
+  coreLength
 }
 
 // a peer with a sparse copy may not hold block 0, but the recorded tail is the

package/lib/topo.js

@@ -9,6 +9,8 @@ const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 
 exports.sort = sort
 exports.replay = replay
+exports.linearize = linearize
+exports.getOplogBatch = getOplogBatch
 
 exports.isLinking = isLinking
 exports.isLinkingAll = isLinkingAll
@@ -92,6 +94,18 @@ async function sort(auto, batch) {
   }
 }
 
+// the order the batches settle in - applying them in it never undoes.
+// batches must arrive after the batches they link
+function linearize(batches) {
+  const list = []
+
+  for (const batch of batches) {
+    addSorted(list, { undo: null, batch, index: -1 })
+  }
+
+  return list.map((e) => e.batch)
+}
+
 function addSorted(list, entry) {
   const node = entry.batch[0]
   const clock = new Clock()
@@ -215,16 +229,17 @@ async function getOplogBatch(undo, core, length, weight, index, readOpts = null)
   const head = createNode(core, length, weight, oplog)
   const legacy = oplog.version <= LEGACY_OPLOG_VERSION
 
-  // if this is legacy the start pointer is always 0, fix
-  if (legacy && !oplog.links.length) {
+  // legacy start is always 0, walk back to the previous head (end 0). links
+  // do not mark the start, legacy batch nodes link their own predecessor.
+  // only the block before the batch can be missing (legacy ff), so a miss is
+  // the boundary
+  if (legacy) {
     for (let i = length - 1; i > 0; i--) {
-      const block = await core.get(i - 1, readOpts)
-      if (block === null) return null // wait: false in replay helper
+      const block = await core.get(i - 1, { wait: false })
+      if (block === null) break
 
-      const oplog = encoding.decodeOplog(block)
-      if (oplog.batch.end === 0) break
+      if (encoding.decodeOplog(block).batch.end === 0) break
       head.batch.start++
-      if (oplog.links.length > 0) break // always start of batch in legacy
     }
   }
 

package/lib/writers.js

@@ -10,6 +10,7 @@ const WRITER_PREFETCH = 10
 
 const BATCH_WRITER_SANITY = 1024
 const BATCH_OPTIMISTIC_SANITY = 32
+const DEFAULT_MAX_UNTRUSTED_WRITERS = 128
 
 const MAX_FORWARD_TIME_DRIFT = 2 * 3600 * 1000
 
@@ -24,6 +25,11 @@ class ActiveWriters {
     this.writable = false
     this.pending = []
 
+    this.maxUntrustedWriters =
+      auto._handlers.maxUntrustedWriters === undefined
+        ? DEFAULT_MAX_UNTRUSTED_WRITERS
+        : auto._handlers.maxUntrustedWriters || Infinity
+
     this._pendingBuffer = new Map()
     this._adding = new Map()
     this._lock = new ReadWriteLock()
@@ -250,6 +256,14 @@ class ActiveWriters {
       let w = this.active.get(id)
       if (w) return w
 
+      // sanity limit while untrusted, a dropped writer is requested again by the
+      // links waiting on it or by peers announcing it once the round has gc'd
+      if (this.active.size >= this.maxUntrustedWriters && !(await this.auto.isLocalTrusted())) {
+        return null
+      }
+
+      this.auto.stats.writersOpened++
+
       const local = b4a.equals(key, this.auto.local.key)
       const encryption = this.auto.encryptionKey ? new WriterEncryption(this.auto) : null
 
@@ -446,6 +460,7 @@ class Writer {
   }
 
   close() {
+    if (!this.isClosed && this !== this.writers.localWriter) this.writers.auto.stats.writersClosed++
     this.isClosed = true
 
     if (this.waiting) {
@@ -677,25 +692,33 @@ class Writer {
 
     const has = await Promise.all(promises)
 
+    let missing = false
+
     for (let i = 0; i < links.length; i++) {
       if (has[i]) continue
       const link = links[i]
-      const id = b4a.toString(link.key, 'hex')
 
-      this.waiting = this.writers.triggers.add(id, link.length, this)
+      // one trigger per writer, wait on the first missing link
+      if (!missing) {
+        const id = b4a.toString(link.key, 'hex')
 
-      // check it didn't arrive underneath us
-      if (await this.system.has(link)) {
-        this.writers.triggers.remove(this.waiting)
-        this.waiting = null
-        continue
+        this.waiting = this.writers.triggers.add(id, link.length, this)
+
+        // check it didn't arrive underneath us
+        if (await this.system.has(link)) {
+          this.writers.triggers.remove(this.waiting)
+          this.waiting = null
+          continue
+        }
+
+        missing = true
       }
 
+      // but wake up every missing link so they fetch in parallel
       this.writers.wakeup(link.key, link.length).catch(noop)
-      return false
     }
 
-    return true
+    return !missing
   }
 
   get appending() {
@@ -773,7 +796,6 @@ function sortsBefore(a, b) {
 }
 
 exports.ActiveWriters = ActiveWriters
-exports.sortsBefore = sortsBefore
 exports.Writer = Writer
 exports.createNode = createNode
 exports.finalizeNodeBatch = finalizeNodeBatch

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.7.3",
+  "version": "2.8.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",