autobee 2.0.0-rc.31 → 2.0.0-rc.34

package/index.js

@@ -129,12 +129,10 @@ module.exports = class Autobee extends ReadyResource {
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
     this._needsUpdate = false
-    this._ackRequired = false
     this._approvalCheck = true
     this._approvedPending = new Map()
     this._approvalRequests = []
     this._prefetchingApprovals = false
-    this._ackedHeads = new Map()
     this._updateLocalCore = null
     this._host = new ApplyCalls(this)
     this._notifyHandler = null
@@ -385,7 +383,6 @@ module.exports = class Autobee extends ReadyResource {
     }
 
     this.local.setEncryption(this._getEncryptionProvider())
-    this.local.setActive(true)
 
     this.writers = new ActiveWriters(this)
 
@@ -464,6 +461,11 @@ module.exports = class Autobee extends ReadyResource {
     for (const head of this.system.heads) {
       await this.writers.add(head.key)
     }
+
+    if (!this.system.heads.length) {
+      await this.writers.add(this.bootstrap.key)
+    }
+
     await this._bump()
   }
 
@@ -554,8 +556,6 @@ module.exports = class Autobee extends ReadyResource {
       else if (head) await this._bootFromHead(head, bootCondition, wait)
     }
 
-    this._ackRequired = false
-
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
@@ -578,16 +578,7 @@ module.exports = class Autobee extends ReadyResource {
             break // revaluate conditions...
           }
 
-          if (await this._bumpPendingWriters()) {
-            this._needsUpdate = true
-            continue
-          }
-
-          if (!this._appendAck()) break
-          this._needsUpdate = true
-        }
-
-        if (!this._interrupting && (await this._appendApprovals())) {
+          if (!(await this._bumpPendingWriters())) break
           this._needsUpdate = true
         }
 
@@ -647,24 +638,44 @@ module.exports = class Autobee extends ReadyResource {
   // wakeup hints keep arriving asynchronously
   async _applyWakeupHints() {
     const hints = this._wakeup.flush()
-    if (!hints.size) return hints
+    if (!hints.size) return []
 
     this.previousDrain = Date.now()
+    const results = await this._filterHints(hints)
 
-    for (const [hex, length] of hints) {
-      const key = b4a.from(hex, 'hex')
+    for (const { key, length } of results) {
       // wakeup() itself no-ops the add for an already-active writer, but still
       // needs to run so it can hint() the announced length - skipping it here
       // for active writers left gc with no hint to protect them
       await this.writers.wakeup(key, length)
     }
 
-    return hints
+    return results
+  }
+
+  async _filterHints(hints) {
+    const results = []
+    const promises = []
+    for (const [hex, length] of hints) {
+      const key = b4a.from(hex, 'hex')
+      promises.push(this._filterTracked(key, length, results))
+    }
+    await Promise.all(promises)
+    return results
+  }
+
+  async _filterTracked(key, length, results) {
+    if (length) {
+      const info = await this.system.get(key)
+      // if we've seen this already, ignore
+      if (info && info.length >= length) return
+    }
+    results.push({ key, length })
   }
 
   async _flushWakeup() {
     const hints = await this._applyWakeupHints()
-    if (!hints.size) return
+    if (!hints.length) return
 
     if (!this._ffEnabled) return
 
@@ -688,10 +699,9 @@ module.exports = class Autobee extends ReadyResource {
   async _readCandidateHeads(hints, timeout) {
     const promises = []
 
-    for (const [hex, length] of hints) {
+    for (const { key, length } of hints) {
       if (length === 0) continue
-      const key = b4a.from(hex, 'hex')
-      promises.push(this._getOplog(key, length, timeout ? { timeout } : null))
+      promises.push(this._resolveOplogHint(key, length, timeout ? { timeout } : null))
     }
 
     const ops = await Promise.all(promises)
@@ -714,6 +724,22 @@ module.exports = class Autobee extends ReadyResource {
     return heads
   }
 
+  async _resolveOplogHint(key, length, opts) {
+    const core = this.openCore(key)
+
+    try {
+      await core.ready()
+      const max = Math.max(length, core.length)
+
+      const block = await this.readOplog(core, max, opts)
+      if (max >= core.length) return block // best one
+
+      return await this.readOplog(core, core.length, opts)
+    } finally {
+      await core.close()
+    }
+  }
+
   async _getOplog(key, length, opts = null) {
     const core = this.openCore(key)
 
@@ -987,51 +1013,6 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _appendApprovals() {
-    if (!this.writers.writable) return false
-
-    const approvals = await this._collectApprovals()
-    if (!approvals) return false
-
-    const links = this.system.getLinks(this.local.key)
-    const t = Math.max(this._now(), this.system.timestamp)
-    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null, approvals)
-    return true
-  }
-
-  // append a null value node to ack writer
-  _appendAck() {
-    if (!this._ackRequired) return false
-    if (!this.writers.writable) return false
-
-    if (this.writers.localWriter.pending !== null) return false
-
-    const links = this.system.getLinks(this.local.key)
-    const t = Math.max(this._now(), this.system.timestamp)
-
-    let unlinked = false
-    for (const { key, length } of links) {
-      const hex = b4a.toString(key, 'hex')
-      const acked = this._ackedHeads.get(hex) || 0
-      if (length > acked) {
-        unlinked = true
-        break
-      }
-    }
-    if (!unlinked) {
-      this._ackRequired = false
-      return false
-    }
-
-    for (const { key, length } of links) {
-      this._ackedHeads.set(b4a.toString(key, 'hex'), length)
-    }
-
-    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null)
-    this._ackRequired = false
-    return true
-  }
-
   async _bumpPendingWriters({ local = false } = {}) {
     if (!local && this._catchupMigratedNodes !== null) {
       const updated = await this._bumpMigratedWriters()
@@ -1398,10 +1379,9 @@ module.exports = class Autobee extends ReadyResource {
 
   async _bootAttempt(head, bootCondition, reference) {
     // peek, so the hints are still there for the drain once we have booted
-    const candidates = await this._readCandidateHeads(
-      this._wakeup.hints,
-      FastForward.DEFAULT_TIMEOUT
-    )
+    const result = await this._filterHints(this._wakeup.hints)
+
+    const candidates = await this._readCandidateHeads(result, FastForward.DEFAULT_TIMEOUT)
 
     const ff = await FastForward.fromHeads(this, [head, ...candidates], {
       force: true,

package/lib/apply-calls.js

@@ -1,5 +1,4 @@
 const ID = require('hypercore-id-encoding')
-const b4a = require('b4a')
 
 class ApplyCalls {
   constructor(auto) {
@@ -44,10 +43,6 @@ class ApplyCalls {
 
   ackWriter(key) {
     if (typeof key === 'string') key = ID.decode(key)
-    // acking an optimistic node means we owe it a linking node
-    if (this.applying && this.applying[0].optimistic && !b4a.equals(key, this.auto.local.key)) {
-      this.auto._ackRequired = true
-    }
     return this.auto.system.ackWriter(key)
   }
 

package/lib/fast-forward.js

@@ -9,10 +9,6 @@ const { LEGACY_AUTOBASE_VERSION } = require('./constants.js')
 const DEFAULT_OP_TIMEOUT = 5_000
 const MIN_FF_GAP = 32
 
-// views are only stamped on the last node of a flush, so a head may sit above
-// the most recent one - how far back we look for it
-const MAX_FLUSH_SEARCH = 64
-
 module.exports = class FastForward {
   constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
     this.auto = auto
@@ -147,18 +143,9 @@ module.exports = class FastForward {
     const core = auto.openCore(head.key)
 
     try {
-      let length = head.length
-
-      for (let i = 0; i < MAX_FLUSH_SEARCH; i++) {
-        const oplog = await auto.readOplog(core, length, opts)
-        if (oplog === null) return null
-        if (oplog.op.views) return oplog
-
-        length = oplog.length - 1
-        if (length <= 0) return null
-      }
-
-      return null
+      const oplog = await auto.readOplog(core, head.length, opts)
+      if (oplog === null) return null
+      return oplog.op.views ? oplog : null
     } finally {
       await core.close()
     }

package/lib/migrations.js

@@ -223,7 +223,9 @@ async function inflateLegacyOplog(buf, core, seq, timeout) {
 
   fetches.push(m.digest.pointer ? core.get(seq - m.digest.pointer, { timeout }) : buf)
   fetches.push(
-    m.checkpoint.pointer ? core.get(seq - m.checkpoint.system.checkpointer, { timeout }) : buf
+    m.checkpoint.system.checkpointer
+      ? core.get(seq - m.checkpoint.system.checkpointer, { timeout })
+      : buf
   )
 
   const [digestNode, checkpointNode] = await Promise.all(fetches)

package/lib/system.js

@@ -71,12 +71,12 @@ module.exports = class Systembee {
       const added = info && info.isAdded !== false
       if (added && !info.isRemoved && length <= info.length && !hint && pending === -1) return
 
-      this.update(key, length, -1, -1, false, true, false, false, -1, hint, pending)
+      this.update(key, length, -1, -1, false, true, false, -1, hint, pending)
       return
     }
 
     const w = isGenesis ? effective : -1
-    this.update(key, length, w, effective, isGenesis, true, false, false, -1, hint, pending)
+    this.update(key, length, w, effective, isGenesis, true, false, -1, hint, pending)
   }
 
   async ackWriter(key, { length = 0 } = {}) {
@@ -84,7 +84,7 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: false })
       length = info ? info.length : 0
     }
-    this.update(key, length, -1, -1, false, false, false, true, -1, null)
+    this.update(key, length, -1, -1, false, false, false, -1, null)
   }
 
   async removeWriter(key, { length = 0 } = {}) {
@@ -92,17 +92,24 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: true })
       length = info ? info.length : 0
     }
-    this.update(key, length, -1, 0, false, false, true, false, -1, null)
+    this.update(key, length, -1, 0, false, false, true, -1, null)
   }
 
   addAnchor(key) {
-    this.update(key, 0, -1, -1, false, true, false, false, -1, null)
+    this.update(key, 0, -1, -1, false, true, false, -1, null)
   }
 
   isGenesis() {
     return this.bee.head() === null || this.bee.head().length === 0
   }
 
+  isHead(key) {
+    for (const head of this.heads) {
+      if (b4a.equals(head.key, key)) return true
+    }
+    return false
+  }
+
   async boot(view, { timeout } = {}) {
     await this.bee.ready()
     this.bee.move(view)
@@ -184,18 +191,7 @@ module.exports = class Systembee {
 
     // resolved sort weight, re-stamped on every reapplication - this is what
     // topo reads back out of the changes stream
-    this.update(
-      node.key,
-      node.length,
-      node.weight,
-      -1,
-      false,
-      false,
-      false,
-      false,
-      node.timestamp,
-      null
-    )
+    this.update(node.key, node.length, node.weight, -1, false, false, false, node.timestamp, null)
   }
 
   async canApply(key, optimistic) {
@@ -236,14 +232,12 @@ module.exports = class Systembee {
         isGenesis: upd.maxWeight !== -1 && !!upd.isGenesis,
         length: upd.length,
         clock: 0,
-        isAdded: upd.isAdded,
-        isAcked: upd.isAcked
+        isAdded: upd.isAdded
       }
     }
 
     if (upd.isAdded) info.isAdded = true
     if (upd.isRemoved) info.isRemoved = true
-    if (upd.isAcked) info.isAcked = true
     if (upd.maxWeight !== -1) {
       info.isGenesis = upd.isGenesis
       info.maxWeight = upd.maxWeight
@@ -284,7 +278,6 @@ module.exports = class Systembee {
     isGenesis,
     isAdded,
     isRemoved,
-    isAcked,
     timestamp,
     grant,
     pending = -1
@@ -308,7 +301,6 @@ module.exports = class Systembee {
         isGenesis: false,
         isRemoved: false,
         isAdded: false,
-        isAcked: false,
         timestamp: 0,
         grant: null,
         pending: -1
@@ -330,9 +322,6 @@ module.exports = class Systembee {
       upd.isRemoved = true
       upd.isAdded = false
     }
-    if (isAcked) {
-      upd.isAcked = true
-    }
     if (length > upd.length) {
       upd.length = length
     }

package/lib/witness.js

@@ -1,6 +1,5 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
-const { DEFAULT_OP_TIMEOUT } = require('./constants.js')
 
 exports.currentWeight = currentWeight
 exports.resolveWeight = resolveWeight
@@ -56,7 +55,7 @@ async function readOplogNode(auto, key, length) {
 
   try {
     await core.ready()
-    const block = await core.get(length - 1, { timeout: DEFAULT_OP_TIMEOUT })
+    const block = await core.get(length - 1)
     return encoding.decodeOplog(block)
   } finally {
     await core.close()

package/lib/writers.js

@@ -18,17 +18,20 @@ class ActiveWriters {
 
     this.auto = auto
     this.active = new Map()
-    this._adding = new Map()
     this.triggers = new Triggers()
-    this.localWriter = new Writer(this, auto.system, true, local, b4a.toString(local.key, 'hex'))
+    this.localWriter = null
     this.writable = false
     this.pending = []
 
     this._pendingBuffer = new Map()
+    this._adding = new Map()
 
+    this.localWriter = new Writer(this, auto.system, true, local, b4a.toString(local.key, 'hex'))
     this.localWriter.attach()
+  }
 
-    this._bootstrap()
+  get size() {
+    return this.active.size
   }
 
   [Symbol.iterator]() {
@@ -195,10 +198,7 @@ class ActiveWriters {
     if (local) await this.updateLocalState()
 
     let w = this.active.get(id)
-    if (w) {
-      w.couple() // couple if we weren't ready first time
-      return w
-    }
+    if (w) return w
 
     // no new sessions once teardown started
     if (this.auto._interrupting) return null
@@ -206,6 +206,7 @@ class ActiveWriters {
     const encryption = this.auto.encryptionKey ? new WriterEncryption(this.auto) : null
 
     const core = this.auto.store.get({
+      active: false,
       key,
       encryption,
       group: local || !this.auto.wakeupCapability ? null : this.auto.wakeupCapability.discoveryKey
@@ -222,7 +223,6 @@ class ActiveWriters {
 
     w = new Writer(this, this.auto.system, false, core, id)
     w.attach()
-    w.couple()
 
     return w
   }
@@ -250,29 +250,6 @@ class ActiveWriters {
     w.decouple()
     if (w !== this.localWriter) await w.close()
   }
-
-  _bootstrap() {
-    const bootstrap = this.auto.bootstrap
-
-    bootstrap.setEncryption(this.auto._getEncryptionProvider())
-    bootstrap.setActive(true)
-
-    const id = b4a.toString(bootstrap.key, 'hex')
-    let w = this.active.get(id)
-    if (w) {
-      w.isBootstrap = true
-      w.couple() // couple if we weren't ready first time
-      return
-    }
-
-    const session = bootstrap.session()
-    session.setEncryption(this.auto._getEncryptionProvider())
-
-    w = new Writer(this, this.auto.system, false, session, id)
-    w.isBootstrap = true // the root of the autobase, never gc'd
-    w.attach()
-    w.couple()
-  }
 }
 
 class Writer {
@@ -298,20 +275,26 @@ class Writer {
     this.isRemoved = false
     this.isCoupled = false
     this.isFrozen = false
-    this.isBootstrap = false
+    this.isHead = false
 
     this._onchangeBound = this._onchange.bind(this)
 
     if (!local) this.core.on('append', this._onchangeBound)
     if (!local) this.core.on('download', this._onchangeBound)
-
-    this.couple()
   }
 
   get isIndexer() {
     return this.weight >= 2
   }
 
+  isSystemHead() {
+    if (this.system.heads.length === 0) {
+      return b4a.equals(this.core.key, this.writers.auto.key)
+    }
+
+    return this.system.isHead(this.core.key)
+  }
+
   couple() {
     if (this.isCoupled) return
     // if we don't have coupler yet we can't add
@@ -357,7 +340,10 @@ class Writer {
 
   // a peer announced this length for us, so we're not caught up until we've seen it
   hint(length) {
-    if (length > this.hintedLength) this.hintedLength = length
+    if (length > this.hintedLength) {
+      this.hintedLength = length
+      this.core.setActive(true)
+    }
   }
 
   _onchange() {
@@ -400,7 +386,7 @@ class Writer {
     // don't drop any local nodes left in buffer, or work a peer told us about
     if (length < this.core.length || length < this.hintedLength || this.appending) {
       this.addPending()
-    } else if (this !== this.writers.localWriter && !this.isIndexer && !this.isBootstrap) {
+    } else if (this !== this.writers.localWriter && !this.isHead) {
       await this.detachAndClose()
     }
   }
@@ -447,12 +433,24 @@ class Writer {
 
   async update() {
     const info = await this.system.get(this.core.key)
+    const length = info ? info.length : 0
 
-    // an anchor core is prologue-only, so once its single node has applied
-    // nothing more can ever come - retire the writer
-    if (info && info.length > 0 && isAnchor(this.core)) {
-      await this.detachAndClose()
-      return info.length
+    const isHead = this.isSystemHead()
+
+    if (isHead) {
+      this.isHead = true
+      this.couple()
+    } else {
+      this.isHead = false
+      this.decouple()
+    }
+
+    const active = length < this.core.length || length < this.hintedLength
+
+    if (this.isHead || active) {
+      this.core.setActive(true)
+    } else {
+      this.core.setActive(false)
     }
 
     if (!info) {
@@ -468,7 +466,7 @@ class Writer {
       this.writers._updateLocalState()
     }
 
-    return info ? info.length : 0
+    return length
   }
 
   next() {
@@ -693,6 +691,7 @@ class Writer {
     head.views = views
 
     head.trusted = trusted ? [{ key: trusted.key, length: trusted.length }] : null
+    this.writers.auto.emit('appending', this.pending)
 
     const buffers = []
     for (let i = 0; i < this.processed; i++) {
@@ -762,7 +761,3 @@ function finalizeNodeBatch(batch) {
 }
 
 function noop() {}
-
-function isAnchor(core) {
-  return !!(core.manifest && core.manifest.signers.length === 0 && core.length === 1)
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.31",
+  "version": "2.0.0-rc.34",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",