autobee 2.0.0-rc.8 → 2.0.0

package/lib/writers.js

@@ -1,7 +1,8 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
 const Triggers = require('./triggers.js')
-const { signAttestation } = require('./witness.js')
+const { LEGACY_OPLOG_VERSION } = require('./constants.js')
+const { resolveWeight } = require('./witness.js')
 const { WriterEncryption } = require('autobee-encryption')
 
 const WRITER_PREFETCH = 10
@@ -17,17 +18,20 @@ class ActiveWriters {
 
     this.auto = auto
     this.active = new Map()
-    this._adding = new Map()
     this.triggers = new Triggers()
-    this.localWriter = new Writer(this, auto.system, true, local, b4a.toString(local.key, 'hex'))
+    this.localWriter = null
     this.writable = false
     this.pending = []
-    this.witnesses = []
-    this.attestations = []
 
+    this._pendingBuffer = new Map()
+    this._adding = new Map()
+
+    this.localWriter = new Writer(this, auto.system, true, local, b4a.toString(local.key, 'hex'))
     this.localWriter.attach()
+  }
 
-    this._bootstrap()
+  get size() {
+    return this.active.size
   }
 
   [Symbol.iterator]() {
@@ -57,24 +61,31 @@ class ActiveWriters {
     let w = this.active.get(id)
 
     if (w === this.localWriter) return
+
+    let added = false
     if (!w) {
-      // anchors are local-only and never replicated - nothing to fetch
-      const info = await this.auto.system.get(key, { unflushed: true })
-      if (info && info.isAnchor) return
       w = await this.add(key)
+      if (!w) return
+      added = true
     }
 
-    // TODO: signal that we've seen length on w for gc
+    const advanced = length > w.hintedLength
+
+    // remember the announced length so gc doesn't drop a writer we haven't caught up on
+    w.hint(length)
+
+    // prefetch the oplog head for the fast-forward that runs right after us
+    if (w && length > 0) w.core.get(length - 1).catch(noop)
 
-    this.bump()
+    if (added || advanced) this.bump()
   }
 
   getLatestLocalOplog() {
     return this.localWriter.getLatest()
   }
 
-  appendLocal(value, timestamp, batch, links, optimistic, witness) {
-    return this.localWriter.append(value, timestamp, batch, links, optimistic, witness)
+  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null) {
+    return this.localWriter.append(value, timestamp, batch, links, optimistic, witness, approvals)
   }
 
   async rotateLocalWriter(core) {
@@ -85,9 +96,6 @@ class ActiveWriters {
     this.localWriter.addPending()
     this.auto.emit('writer', this.localWriter)
 
-    // old local key witnesses are useless
-    this.witnesses = []
-
     await this.updateLocalState()
   }
 
@@ -95,19 +103,21 @@ class ActiveWriters {
     return this.localWriter.clear()
   }
 
-  flushLocal(views) {
+  async flushLocal(views) {
     const w = this.localWriter
 
     // only consume the queue when a flush will actually write
-    if (w.pending === null || w.processed === 0) return Promise.resolve()
+    if (w.pending === null || w.processed === 0) return
 
-    const queued = this.attestations
-    this.attestations = []
+    const trusted = await this.auto.trusted.mostRecentTrusted(this.auto._workingView.view, null)
 
-    return w.flush(views, queued)
+    return w.flush(views, trusted)
   }
 
   async refresh() {
+    // peeked batches were read against the pre-reboot system, drop them
+    this._pendingBuffer.clear()
+
     if (this.localWriter && !this.localWriter.isClosed) {
       await this.updateLocalState()
     }
@@ -122,6 +132,37 @@ class ActiveWriters {
     await this.localWriter.update()
   }
 
+  async nextPendingNode({ local = false } = {}) {
+    for (const w of this.pending.slice()) {
+      if (local && w !== this.localWriter) continue
+      if (this._pendingBuffer.has(w) || w.isFrozen) continue
+
+      const batch = await w.next()
+      if (batch !== null) this._pendingBuffer.set(w, batch)
+    }
+
+    if (this._pendingBuffer.size === 0) return null
+
+    let writer = null
+    let best = null
+
+    for (const [w, batch] of this._pendingBuffer) {
+      if (local && w !== this.localWriter) continue
+      const head = batch[0]
+      // -1 marks a node next() has not resolved a sort weight for yet
+      if (head.weight < 0) head.weight = await resolveWeight(this.auto, head)
+      if (best === null || sortsBefore(head, best[0])) {
+        writer = w
+        best = batch
+      }
+    }
+
+    if (writer === null) return null
+
+    this._pendingBuffer.delete(writer)
+    return { writer, batch: best }
+  }
+
   _updateLocalState() {
     const w = this.localWriter
 
@@ -157,10 +198,7 @@ class ActiveWriters {
     if (local) await this.updateLocalState()
 
     let w = this.active.get(id)
-    if (w) {
-      w.couple() // couple if we weren't ready first time
-      return w
-    }
+    if (w) return w
 
     // no new sessions once teardown started
     if (this.auto._interrupting) return null
@@ -168,6 +206,7 @@ class ActiveWriters {
     const encryption = this.auto.encryptionKey ? new WriterEncryption(this.auto) : null
 
     const core = this.auto.store.get({
+      active: false,
       key,
       encryption,
       group: local || !this.auto.wakeupCapability ? null : this.auto.wakeupCapability.discoveryKey
@@ -184,7 +223,6 @@ class ActiveWriters {
 
     w = new Writer(this, this.auto.system, false, core, id)
     w.attach()
-    w.couple()
 
     return w
   }
@@ -196,42 +234,22 @@ class ActiveWriters {
     const w = this.active.get(id)
     if (!w) return
 
+    // nodes this writer appended before the removal may still sort before it,
+    // and their ordering advances its record everywhere - closing the session
+    // while we are behind strands us one node short forever. reset() closes it
+    // once we have caught up
+    if (w !== this.localWriter) {
+      const length = await w.update()
+      if (length < w.core.length || length < w.hintedLength) {
+        w.addPending()
+        return
+      }
+    }
+
     w.detach()
     w.decouple()
     if (w !== this.localWriter) await w.close()
   }
-
-  _bootstrap() {
-    const bootstrap = this.auto.bootstrap
-
-    bootstrap.setEncryption(this.auto._getEncryptionProvider())
-    bootstrap.setActive(true)
-
-    const id = b4a.toString(bootstrap.key, 'hex')
-    let w = this.active.get(id)
-    if (w) {
-      w.couple() // couple if we weren't ready first time
-      return
-    }
-
-    const session = bootstrap.session()
-    session.setEncryption(this.auto._getEncryptionProvider())
-
-    w = new Writer(this, this.auto.system, false, session, id)
-    w.attach()
-    w.couple()
-  }
-
-  attest(witnessed) {
-    const local = this.localWriter
-
-    for (const { key, weight } of witnessed) {
-      if (b4a.equals(key, local.core.key)) continue // never attest about myself
-      if (this.attestations.some((a) => a.weight === weight && b4a.equals(a.key, key))) continue
-
-      this.attestations.push({ key, weight })
-    }
-  }
 }
 
 class Writer {
@@ -248,6 +266,7 @@ class Writer {
     this.appendLength = this.core.length + 1
     this.waiting = null
     this.download = null
+    this.hintedLength = 0
 
     this.isPending = false
     this.isClosed = false
@@ -256,17 +275,26 @@ class Writer {
     this.isRemoved = false
     this.isCoupled = false
     this.isFrozen = false
+    this.isHead = false
 
     this._onchangeBound = this._onchange.bind(this)
 
     if (!local) this.core.on('append', this._onchangeBound)
     if (!local) this.core.on('download', this._onchangeBound)
-
-    this.couple()
   }
 
   get isIndexer() {
-    return this.weight === 2
+    return this.weight >= 2
+  }
+
+  isSystemHead() {
+    if (this.core.length === 0) return false
+
+    if (this.system.heads.length === 0) {
+      return b4a.equals(this.core.key, this.writers.auto.key)
+    }
+
+    return this.system.isHead(this.core.key)
   }
 
   couple() {
@@ -303,6 +331,8 @@ class Writer {
     if (!this.isPending) return
     this.isPending = false
 
+    this.writers._pendingBuffer.delete(this)
+
     const head = this.writers.pending.pop()
     if (head !== this) {
       this.writers.pending[this.index] = head
@@ -310,7 +340,16 @@ class Writer {
     }
   }
 
+  // a peer announced this length for us, so we're not caught up until we've seen it
+  hint(length) {
+    if (length > this.hintedLength) {
+      this.hintedLength = length
+      this.core.setActive(true)
+    }
+  }
+
   _onchange() {
+    this.hint(this.core.length)
     if (this.waiting === null) this.bump()
   }
 
@@ -347,10 +386,10 @@ class Writer {
 
     const length = await this.update()
 
-    // don't drop any local nodes left in buffer
-    if (length < this.core.length || this.appending) {
+    // don't drop any local nodes left in buffer, or work a peer told us about
+    if (length < this.core.length || length < this.hintedLength || this.appending) {
       this.addPending()
-    } else if (this !== this.writers.localWriter && !this.isIndexer) {
+    } else if (this !== this.writers.localWriter && !this.isHead) {
       await this.detachAndClose()
     }
   }
@@ -370,6 +409,7 @@ class Writer {
 
   detachAndClose() {
     this.detach()
+    this.decouple()
     return this.close()
   }
 
@@ -396,10 +436,24 @@ class Writer {
 
   async update() {
     const info = await this.system.get(this.core.key)
+    const length = info ? info.length : 0
 
-    if (info && info.isAnchor && this !== this.writers.localWriter) {
-      await this.detachAndClose()
-      return info.length
+    const isHead = this.isSystemHead()
+
+    if (isHead) {
+      this.isHead = true
+      this.couple()
+    } else {
+      this.isHead = false
+      this.decouple()
+    }
+
+    const active = length < this.core.length || length < this.hintedLength
+
+    if (this.isHead || active) {
+      this.core.setActive(true)
+    } else {
+      this.core.setActive(false)
     }
 
     if (!info) {
@@ -415,7 +469,7 @@ class Writer {
       this.writers._updateLocalState()
     }
 
-    return info ? info.length : 0
+    return length
   }
 
   next() {
@@ -493,30 +547,8 @@ class Writer {
         return null
       }
 
-      for (const att of node.attestations || []) {
-        if (!b4a.equals(att.key, this.writers.localWriter.core.key)) continue
-
-        const seen = this.writers.witnesses.some(
-          (w) =>
-            w.length === node.length &&
-            w.attestation.weight === att.weight &&
-            b4a.equals(w.key, node.key)
-        )
-        if (seen) continue
-
-        const manifest = this.core.getManifest({ raw: true })
-        if (!manifest) continue
-
-        this.writers.witnesses.push({
-          key: node.key,
-          length: node.length,
-          attestation: att,
-          manifest
-        })
-      }
-
       batch.push(node)
-      if (b.end === 0) return batch
+      if (b.end === 0) return finalizeNodeBatch(batch)
 
       // TODO: mark as dead
       if (batch.length >= BATCH_WRITER_SANITY) return null
@@ -572,17 +604,24 @@ class Writer {
   }
 
   async _isLinked(node) {
-    // the backer gates like a link: present-or-wait
     let links = node.links
     if (node.witness) {
       links = links.slice()
-      links.push({ key: node.witness.backer.key, length: node.witness.backer.length })
+      links.push({ key: node.witness.link.key, length: node.witness.link.length })
     }
 
     const promises = new Array(links.length)
 
     for (let i = 0; i < links.length; i++) {
-      promises[i] = this.system.has(links[i])
+      const link = links[i]
+
+      // legacy batch nodes link their own previous node, satisfied by oplog order
+      if (link.length < node.length && b4a.equals(link.key, this.core.key)) {
+        promises[i] = true
+        continue
+      }
+
+      promises[i] = this.system.has(link)
     }
 
     const has = await Promise.all(promises)
@@ -612,16 +651,18 @@ class Writer {
     return this.pending !== null
   }
 
-  append(value, timestamp, batch, links, optimistic, witness) {
+  append(value, timestamp, batch, links, optimistic, witness, approvals = null) {
     if (this.pending === null) this.pending = []
 
     const oplog = {
       version: encoding.OPLOG_VERSION,
       timestamp,
       links,
-      witness: witness || null,
       batch,
       views: null,
+      trusted: null,
+      witness: witness || null,
+      approvals,
       optimistic: optimistic && !this.writers.writable,
       value
     }
@@ -646,39 +687,43 @@ class Writer {
     this.processed = 0
   }
 
-  flush(views, queued) {
-    if (this.pending === null || this.processed === 0) return Promise.resolve()
+  async flush(views, trusted) {
+    if (this.pending === null || this.processed === 0) return
 
     const head = this.pending[this.processed - 1]
     head.views = views
 
-    if (queued && queued.length) {
-      const attestations = []
-      for (const { key, weight } of queued) {
-        const signature = signAttestation(
-          this.core.keyPair.secretKey,
-          this.core.key,
-          key,
-          head.length,
-          weight
-        )
-        attestations.push({ key, weight, signature })
-      }
-      head.attestations = attestations
-    }
+    head.trusted = trusted ? [{ key: trusted.key, length: trusted.length }] : null
+    this.writers.auto.emit('appending', this.pending)
 
     const buffers = []
     for (let i = 0; i < this.processed; i++) {
       buffers.push(encoding.encodeOplog(this.pending[i]))
     }
-    this.clear()
-    return this.core.append(buffers)
+
+    try {
+      await this.core.append(buffers)
+    } finally {
+      this.clear()
+    }
   }
 }
 
+// mirrors topo.cmp - kept local because topo.js requires this module
+function sortsBefore(a, b) {
+  if (a.weight !== b.weight) return a.weight > b.weight
+  if (a.timestamp !== b.timestamp) return a.timestamp < b.timestamp
+
+  const c = b4a.compare(a.key, b.key)
+  if (c !== 0) return c < 0
+
+  return a.length < b.length
+}
+
 exports.ActiveWriters = ActiveWriters
 exports.Writer = Writer
 exports.createNode = createNode
+exports.finalizeNodeBatch = finalizeNodeBatch
 
 function createNode(core, length, weight, oplog) {
   return {
@@ -690,13 +735,32 @@ function createNode(core, length, weight, oplog) {
     version: oplog.version,
     timestamp: oplog.timestamp,
     links: oplog.links,
-    witness: oplog.witness || null,
-    attestations: oplog.attestations || null,
     batch: oplog.batch,
     views: oplog.views,
+    trusted: oplog.trusted || [],
+    witness: oplog.witness || null,
+    approvals: oplog.approvals || null,
     optimistic: oplog.optimistic,
     value: oplog.value
   }
 }
 
+function finalizeNodeBatch(batch) {
+  // replay batches keep unreachable blocks as null holes (topo.getOplogBatch)
+  let first = null
+  for (const node of batch) {
+    if (node !== null) {
+      first = node
+      break
+    }
+  }
+  if (first === null || first.version > LEGACY_OPLOG_VERSION) return batch
+
+  for (let i = 0; i < batch.length; i++) {
+    if (batch[i] !== null) batch[i].batch.start = i
+  }
+
+  return batch
+}
+
 function noop() {}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.8",
+  "version": "2.0.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -26,7 +26,7 @@
     "autobee-encryption": "^0.1.2",
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
-    "compact-encoding": "^2.18.0",
+    "compact-encoding": "^3.3.0",
     "hyperbee2": "^2.10.3",
     "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",
@@ -38,11 +38,12 @@
   },
   "devDependencies": {
     "brittle": "^4.1.0",
-    "corestore": "^7.11.0",
+    "corestore": "^7.12.2",
     "hypercore-crypto": "^3.6.1",
     "lunte": "^1.6.0",
     "prettier": "^3.8.1",
     "prettier-config-holepunch": "^2.0.0",
-    "protomux-wakeup": "^2.9.0"
+    "protomux-wakeup": "^2.9.0",
+    "replication-simulator": "^3.0.0"
   }
 }