autobee 2.0.0-rc.2 → 2.0.0-rc.21

package/lib/topo.js

@@ -6,6 +6,8 @@ const asserts = require('./asserts.js')
 const EMPTY_HEAD = { length: 0, key: null }
 const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 
+const DEFAULT_OP_TIMEOUT = 5000
+
 exports.sort = sort
 exports.rollback = rollback
 exports.replay = replay
@@ -13,6 +15,7 @@ exports.getMostRecentHead = getMostRecentHead
 
 exports.isLinking = isLinking
 exports.isLinkingAll = isLinkingAll
+exports.isLegacyNode = isLegacyNode
 
 class Clock {
   constructor() {
@@ -35,26 +38,46 @@ class Clock {
 async function getMostRecentHead(auto, system) {
   const ch = system.createChangesStream()
   for await (const data of ch) {
-    const { ref } = getOplog(data)
-    return { key: ref.oplog.key, length: ref.oplog.length }
+    const refs = getOplogCompat(data)
+    if (!refs.length) continue
+
+    const { oplog } = refs[0]
+    return { key: oplog.key, length: oplog.length }
   }
+
+  return null
 }
 
 async function rollback(auto, system, verified) {
+  if (!(await isApplied(system, verified.op))) return null
+
   const tip = []
   const ch = system.createChangesStream()
 
+  let found = false
+
   for await (const data of ch) {
     const { ref, info } = getOplog(data)
+
     if (sameNode(ref.oplog, verified.op)) {
-      asserts.assert(info.flushes === verified.flushes, 'Expect tip to be continuous')
+      // stale views.flushes - the verified head describes a prefix this system
+      // does not share, so nothing above it is ours to reapply
+      if (info.flushes === verified.flushes) found = true
+      break
+    }
+
+    // walking back, a writer's lengths only decrease - once this writer is
+    // behind the verified head the marker cannot be further down
+    if (b4a.equals(ref.oplog.key, verified.op.key) && ref.oplog.length < verified.op.length) {
       break
     }
 
     tip.push(ref)
   }
 
-  const inflated = await inflate(auto, tip)
+  if (!found) return null
+
+  const inflated = await inflate(auto, tip.reverse())
 
   // @todo: verify sort
 
@@ -65,17 +88,32 @@ async function rollback(auto, system, verified) {
   }
 }
 
+// nodes we cannot resolve are returned as null rather than dropped, so callers
+// can tell an incomplete replay from a short one
 async function replay(auto) {
-  const ch = auto.system.bee.createChangesStream()
   const replay = []
 
-  for await (const data of ch) {
-    const { ref } = getOplog(data)
-    replay.push(ref)
+  for await (const entry of oplogIterator(auto.system.bee)) {
+    // the walk was cut short, nothing below this point is reachable
+    if (entry === null) {
+      replay.push(null)
+      break
+    }
+
+    for (const ref of entry.refs) replay.push(ref)
   }
 
-  const inflated = await inflate(auto, replay.reverse())
-  return inflated.flatMap((b) => b.batch)
+  // same reasoning as the iterator - never block on what we do not have
+  const inflated = await inflate(auto, replay.reverse(), { wait: false })
+  return inflated.flatMap((b) => (b === null ? [null] : b.batch))
+}
+
+async function isApplied(system, node) {
+  const entry = await system.get(encoding.encodeSystemWriterKey(node.key))
+  if (entry === null) return false
+
+  const info = encoding.decodeSystemWriter(entry.key, entry.value)
+  return info.length >= node.length
 }
 
 async function sort(auto, batch) {
@@ -192,12 +230,21 @@ function cmp(a, b) {
   return a.length < b.length ? -1 : 1
 }
 
-async function inflate(auto, reorder) {
+async function inflate(auto, reorder, opts = null) {
+  const readOpts = opts === null ? { timeout: DEFAULT_OP_TIMEOUT } : opts
+
   const promises = []
   const sessions = new Map()
 
   for (let i = 0; i < reorder.length; i++) {
     const ref = reorder[i]
+
+    // marker for a node we could not resolve, pass it through
+    if (ref === null) {
+      promises.push(null)
+      continue
+    }
+
     const id = b4a.toString(ref.oplog.key, 'hex')
     // only seeds the node - reapplication re-resolves from the witness
     const weight = ref.oplog.weight
@@ -214,7 +261,7 @@ async function inflate(auto, reorder) {
       sessions.set(id, core)
     }
 
-    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i))
+    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i, readOpts))
   }
 
   try {
@@ -224,9 +271,10 @@ async function inflate(auto, reorder) {
   }
 }
 
-async function getOplogBatch(undo, core, length, weight, index) {
+async function getOplogBatch(undo, core, length, weight, index, opts = null) {
   const seq = length - 1
-  const block = await core.get(seq)
+  const block = await core.get(seq, opts)
+  if (block === null) return null
   const oplog = encoding.decodeOplog(block)
   const head = createNode(core, length, weight, oplog)
 
@@ -238,12 +286,18 @@ async function getOplogBatch(undo, core, length, weight, index) {
   const end = seq + head.batch.end // skip head, we have it
 
   for (let i = start; i < end; i++) {
-    remaining.push(core.get(i))
+    remaining.push(core.get(i, opts))
   }
 
   if (remaining.length) {
     const blocks = await Promise.all(remaining)
     for (let i = 0; i < blocks.length; i++) {
+      // keep the hole so the gap in the batch stays visible
+      if (blocks[i] === null) {
+        batch.push(null)
+        continue
+      }
+
       const oplog = encoding.decodeOplog(blocks[i])
       batch.push(createNode(core, i + start + 1, weight, oplog))
     }
@@ -254,8 +308,14 @@ async function getOplogBatch(undo, core, length, weight, index) {
 }
 
 function getLocalBatch(undo, writer, length, weight, index) {
+  if (writer.core.closed) throw new Error('Writer core is closed')
+
   if (length > writer.core.length) {
-    if (!writer.pending) throw new Error('No local nodes')
+    if (!writer.pending) {
+      throw new Error(
+        `No local nodes: looking for ${length}, found ${writer.core.length}@${writer.core.id}`
+      )
+    }
 
     let i = 0
     while (i < writer.pending.length) {
@@ -264,9 +324,10 @@ function getLocalBatch(undo, writer, length, weight, index) {
     }
 
     const head = writer.pending[i]
+    const headBatch = head.batch || { start: 0, end: 0 }
 
-    const start = i - head.batch.start
-    const end = i + head.batch.end + 1 // include head
+    const start = i - headBatch.start
+    const end = i + headBatch.end + 1 // include head
 
     const batch = writer.pending.slice(start, end)
     return { undo, batch, index }
@@ -275,6 +336,19 @@ function getLocalBatch(undo, writer, length, weight, index) {
   return getOplogBatch(undo, writer.core, length, weight, index)
 }
 
+// tolerates legacy nodes instead of asserting
+async function* oplogIterator(bee) {
+  try {
+    for await (const node of bee.createChangesStream({ wait: false })) {
+      const legacy = isLegacyNode(node)
+      const refs = getOplogCompat(node)
+      if (refs.length) yield { node, refs, legacy }
+    }
+  } catch {
+    yield null // signal data not available
+  }
+}
+
 function getOplog(data) {
   const result = { ref: null, info: null }
   for (const { keys } of data.batch) {
@@ -300,6 +374,35 @@ function getOplog(data) {
   asserts.bail('Bad system node')
 }
 
+// legacy nodes carry no isOplog flag so we cannot tell which writer update is
+// the oplog to fetch - yield every candidate until we have a better signal
+function getOplogCompat(data) {
+  const undo = data.tail || EMPTY_HEAD
+  const oplogs = []
+  const candidates = []
+
+  for (const { keys } of data.batch) {
+    for (const k of keys) {
+      if (k.key[0] !== 1) continue
+
+      let value = null
+      try {
+        value = encoding.decodeSystemWriter(k.key, k.value)
+      } catch {
+        continue
+      }
+
+      // writer was just added, nothing to fetch
+      if (!value.length) continue
+
+      if (value.isOplog) oplogs.push({ undo, oplog: value })
+      else candidates.push({ undo, oplog: value })
+    }
+  }
+
+  return oplogs.length ? oplogs : candidates
+}
+
 function isLinking(node, link) {
   if (node.length >= link.length && b4a.equals(node.key, link.key)) {
     return true

package/lib/trusted.js

@@ -0,0 +1,44 @@
+const b4a = require('b4a')
+
+class TrustedPeers {
+  constructor(hooks) {
+    this.trusted = new Map()
+    this.pending = new Map()
+    this.hooks = hooks
+  }
+
+  async isTrusted(key, reference) {
+    const hex = b4a.toString(key, 'hex')
+    if (this.trusted.get(hex)) return true
+
+    let promise = this.pending.get(hex)
+    if (promise) return promise
+
+    promise = this.hooks.isTrusted
+      ? this.hooks.isTrusted(key, reference)
+      : Promise.resolve(!this.hooks.mostRecentTrusted)
+    this.pending.set(hex, promise)
+
+    const trusted = await promise
+    if (trusted) this.trusted.set(hex, trusted)
+
+    if (this.pending.get(hex) === promise) this.pending.delete(hex)
+    return trusted
+  }
+
+  read(trusted) {
+    return trusted || null
+  }
+
+  mostRecentTrusted(target, reference) {
+    if (!this.hooks.mostRecentTrusted) return Promise.resolve(null)
+    return this.hooks.mostRecentTrusted(target, reference)
+  }
+
+  clear() {
+    this.pending.clear()
+    this.trusted.clear()
+  }
+}
+
+module.exports = TrustedPeers

package/lib/updates.js

@@ -1,5 +1,3 @@
-const b4a = require('b4a')
-
 module.exports = class UpdateChanges {
   constructor(auto) {
     this.auto = auto
@@ -21,57 +19,22 @@ module.exports = class UpdateChanges {
         flushes: this.auto.system.flushes,
         from: this.tracking[k].head(),
         to: null,
-        shared: null
+        incremental: false
       })
     }
   }
 
   finalise() {
-    const floor = this.auto.system.shared
-
     // flush count at `to` - safe to read live, state is frozen for the hook
     const flushes = this.auto.system.flushes
 
     for (const k in this.tracking) {
       const tracked = this.byName.get(k)
-      const startFlushes = tracked.flushes
 
       tracked.to = this.tracking[k].head()
       tracked.flushes = flushes
-
-      if (floor && floor.flushes < startFlushes) {
-        tracked.shared = k === 'view' ? floor.view : floor.system
-      } else {
-        tracked.shared = tracked.from
-      }
     }
 
     return this.byName
   }
-
-  static from(shared, current) {
-    if (
-      shared.flushes === current.flushes &&
-      shared.system.length === current.system.length &&
-      (shared.system.key === current.system.key ||
-        (shared.system.key &&
-          current.system.key &&
-          b4a.equals(shared.system.key, current.system.key)))
-    ) {
-      return null
-    }
-
-    const byName = new Map()
-    const flushes = current.flushes
-
-    byName.set('view', { from: shared.view, shared: shared.view, to: current.view, flushes })
-    byName.set('system', {
-      from: shared.system,
-      shared: shared.system,
-      to: current.system,
-      flushes
-    })
-
-    return byName
-  }
 }

package/lib/writers.js

@@ -1,7 +1,7 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
 const Triggers = require('./triggers.js')
-const { signAttestation } = require('./witness.js')
+const { signAttestation, resolveWeight } = require('./witness.js')
 const { WriterEncryption } = require('autobee-encryption')
 
 const WRITER_PREFETCH = 10
@@ -25,6 +25,8 @@ class ActiveWriters {
     this.witnesses = []
     this.attestations = []
 
+    this._pendingBuffer = new Map()
+
     this.localWriter.attach()
 
     this._bootstrap()
@@ -62,9 +64,14 @@ class ActiveWriters {
       const info = await this.auto.system.get(key, { unflushed: true })
       if (info && info.isAnchor) return
       w = await this.add(key)
+      if (!w) return
     }
 
-    // TODO: signal that we've seen length on w for gc
+    // remember the announced length so gc doesn't drop a writer we haven't caught up on
+    w.hint(length)
+
+    // prefetch the oplog head for the fast-forward that runs right after us
+    if (w && length > 0) w.core.get(length - 1).catch(noop)
 
     this.bump()
   }
@@ -95,19 +102,24 @@ class ActiveWriters {
     return this.localWriter.clear()
   }
 
-  flushLocal(views) {
+  async flushLocal(views) {
     const w = this.localWriter
 
     // only consume the queue when a flush will actually write
-    if (w.pending === null || w.processed === 0) return Promise.resolve()
+    if (w.pending === null || w.processed === 0) return
 
     const queued = this.attestations
     this.attestations = []
 
-    return w.flush(views, queued)
+    const trusted = await this.auto.trusted.mostRecentTrusted(this.auto._workingView.view, null)
+
+    return w.flush(views, queued, trusted)
   }
 
   async refresh() {
+    // peeked batches were read against the pre-reboot system, drop them
+    this._pendingBuffer.clear()
+
     if (this.localWriter && !this.localWriter.isClosed) {
       await this.updateLocalState()
     }
@@ -122,6 +134,33 @@ class ActiveWriters {
     await this.localWriter.update()
   }
 
+  async nextPendingNode() {
+    for (const w of this.pending.slice()) {
+      if (this._pendingBuffer.has(w) || w.isFrozen) continue
+
+      const batch = await w.next()
+      if (batch !== null) this._pendingBuffer.set(w, batch)
+    }
+
+    if (this._pendingBuffer.size === 0) return null
+
+    let writer = null
+    let best = null
+
+    for (const [w, batch] of this._pendingBuffer) {
+      const head = batch[0]
+      // -1 marks a node next() has not resolved a sort weight for yet
+      if (head.weight < 0) head.weight = await resolveWeight(this.auto, head)
+      if (best === null || sortsBefore(head, best[0])) {
+        writer = w
+        best = batch
+      }
+    }
+
+    this._pendingBuffer.delete(writer)
+    return { writer, batch: best }
+  }
+
   _updateLocalState() {
     const w = this.localWriter
 
@@ -210,11 +249,16 @@ class ActiveWriters {
     const id = b4a.toString(bootstrap.key, 'hex')
     let w = this.active.get(id)
     if (w) {
+      w.isBootstrap = true
       w.couple() // couple if we weren't ready first time
       return
     }
 
-    w = new Writer(this, this.auto.system, false, bootstrap, id)
+    const session = bootstrap.session()
+    session.setEncryption(this.auto._getEncryptionProvider())
+
+    w = new Writer(this, this.auto.system, false, session, id)
+    w.isBootstrap = true // the root of the autobase, never gc'd
     w.attach()
     w.couple()
   }
@@ -245,6 +289,7 @@ class Writer {
     this.appendLength = this.core.length + 1
     this.waiting = null
     this.download = null
+    this.hintedLength = 0
 
     this.isPending = false
     this.isClosed = false
@@ -253,6 +298,7 @@ class Writer {
     this.isRemoved = false
     this.isCoupled = false
     this.isFrozen = false
+    this.isBootstrap = false
 
     this._onchangeBound = this._onchange.bind(this)
 
@@ -300,6 +346,8 @@ class Writer {
     if (!this.isPending) return
     this.isPending = false
 
+    this.writers._pendingBuffer.delete(this)
+
     const head = this.writers.pending.pop()
     if (head !== this) {
       this.writers.pending[this.index] = head
@@ -307,6 +355,11 @@ class Writer {
     }
   }
 
+  // a peer announced this length for us, so we're not caught up until we've seen it
+  hint(length) {
+    if (length > this.hintedLength) this.hintedLength = length
+  }
+
   _onchange() {
     if (this.waiting === null) this.bump()
   }
@@ -344,10 +397,10 @@ class Writer {
 
     const length = await this.update()
 
-    // don't drop any local nodes left in buffer
-    if (length < this.core.length || this.appending) {
+    // don't drop any local nodes left in buffer, or work a peer told us about
+    if (length < this.core.length || length < this.hintedLength || this.appending) {
       this.addPending()
-    } else if (this !== this.writers.localWriter && !this.isIndexer) {
+    } else if (this !== this.writers.localWriter && !this.isIndexer && !this.isBootstrap) {
       await this.detachAndClose()
     }
   }
@@ -367,6 +420,7 @@ class Writer {
 
   detachAndClose() {
     this.detach()
+    this.decouple()
     return this.close()
   }
 
@@ -620,7 +674,8 @@ class Writer {
       batch,
       views: null,
       optimistic: optimistic && !this.writers.writable,
-      value
+      value,
+      trusted: null
     }
 
     const node = createNode(this.core, this.appendLength++, -1, oplog)
@@ -643,12 +698,16 @@ class Writer {
     this.processed = 0
   }
 
-  flush(views, queued) {
-    if (this.pending === null || this.processed === 0) return Promise.resolve()
+  async flush(views, queued, trusted) {
+    if (this.pending === null || this.processed === 0) return
 
     const head = this.pending[this.processed - 1]
     head.views = views
 
+    head.trusted = trusted
+      ? [{ key: trusted.key, length: trusted.length, flushes: views.flushes }]
+      : null
+
     if (queued && queued.length) {
       const attestations = []
       for (const { key, weight } of queued) {
@@ -668,11 +727,26 @@ class Writer {
     for (let i = 0; i < this.processed; i++) {
       buffers.push(encoding.encodeOplog(this.pending[i]))
     }
-    this.clear()
-    return this.core.append(buffers)
+
+    try {
+      await this.core.append(buffers)
+    } finally {
+      this.clear()
+    }
   }
 }
 
+// mirrors topo.cmp - kept local because topo.js requires this module
+function sortsBefore(a, b) {
+  if (a.weight !== b.weight) return a.weight > b.weight
+  if (a.timestamp !== b.timestamp) return a.timestamp < b.timestamp
+
+  const c = b4a.compare(a.key, b.key)
+  if (c !== 0) return c < 0
+
+  return a.length < b.length
+}
+
 exports.ActiveWriters = ActiveWriters
 exports.Writer = Writer
 exports.createNode = createNode
@@ -692,7 +766,8 @@ function createNode(core, length, weight, oplog) {
     batch: oplog.batch,
     views: oplog.views,
     optimistic: oplog.optimistic,
-    value: oplog.value
+    value: oplog.value,
+    trusted: oplog.trusted || []
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.2",
+  "version": "2.0.0-rc.21",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -26,7 +26,7 @@
     "autobee-encryption": "^0.1.2",
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
-    "compact-encoding": "^2.18.0",
+    "compact-encoding": "^3.3.0",
     "hyperbee2": "^2.10.3",
     "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",