autobee 2.0.0-rc.8 → 2.0.0

package/lib/topo.js

@@ -1,13 +1,15 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
-const { createNode } = require('./writers.js')
+const { createNode, finalizeNodeBatch } = require('./writers.js')
 const asserts = require('./asserts.js')
+const { LEGACY_OPLOG_VERSION } = require('./constants.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 
+const DEFAULT_OP_TIMEOUT = 5000
+
 exports.sort = sort
-exports.rollback = rollback
 exports.replay = replay
 exports.getMostRecentHead = getMostRecentHead
 
@@ -36,47 +38,34 @@ class Clock {
 async function getMostRecentHead(auto, system) {
   const ch = system.createChangesStream()
   for await (const data of ch) {
-    const { ref } = getOplog(data)
-    return { key: ref.oplog.key, length: ref.oplog.length }
-  }
-}
+    const refs = getOplogCompat(data)
+    if (!refs.length) continue
 
-async function rollback(auto, system, verified) {
-  const tip = []
-  const ch = system.createChangesStream()
-
-  for await (const data of ch) {
-    const { ref, info } = getOplog(data)
-    if (sameNode(ref.oplog, verified.op)) {
-      asserts.assert(info.flushes === verified.flushes, 'Expect tip to be continuous')
-      break
-    }
-
-    tip.push(ref)
+    const { oplog } = refs[0]
+    return { key: oplog.key, length: oplog.length }
   }
 
-  const inflated = await inflate(auto, tip)
-
-  // @todo: verify sort
-
-  return {
-    undo: null,
-    view: null,
-    tip: inflated.map(getBatch)
-  }
+  return null
 }
 
+// nodes we cannot resolve are returned as null rather than dropped, so callers
+// can tell an incomplete replay from a short one
 async function replay(auto) {
-  const ch = auto.system.bee.createChangesStream()
   const replay = []
 
-  for await (const data of ch) {
-    const { ref } = getOplog(data)
-    replay.push(ref)
+  for await (const entry of oplogIterator(auto.system.bee)) {
+    // the walk was cut short, nothing below this point is reachable
+    if (entry === null) {
+      replay.push(null)
+      break
+    }
+
+    for (const ref of entry.refs) replay.push(ref)
   }
 
-  const inflated = await inflate(auto, replay.reverse())
-  return inflated.flatMap((b) => b.batch)
+  // same reasoning as the iterator - never block on what we do not have
+  const inflated = await inflate(auto, replay.reverse(), { wait: false })
+  return inflated.flatMap((b) => (b === null ? [null] : b.batch))
 }
 
 async function sort(auto, batch) {
@@ -87,7 +76,7 @@ async function sort(auto, batch) {
   for await (const data of ch) {
     if (isLegacyNode(data)) break
 
-    const { ref, info } = getOplog(data)
+    const { ref } = getOplog(data)
     const link = {
       key: ref.oplog.key,
       length: ref.oplog.length,
@@ -129,8 +118,7 @@ function addSorted(list, entry) {
     clock.add(link)
   }
 
-  // backer MUST precede node
-  if (node.witness) clock.add(node.witness.backer)
+  if (node.witness) clock.add(node.witness.link)
 
   const tip = []
   const linked = []
@@ -193,12 +181,21 @@ function cmp(a, b) {
   return a.length < b.length ? -1 : 1
 }
 
-async function inflate(auto, reorder) {
+async function inflate(auto, reorder, { wait = true, timeout = DEFAULT_OP_TIMEOUT } = {}) {
+  const readOpts = wait ? { timeout } : { wait: false }
+
   const promises = []
   const sessions = new Map()
 
   for (let i = 0; i < reorder.length; i++) {
     const ref = reorder[i]
+
+    // marker for a node we could not resolve, pass it through
+    if (ref === null) {
+      promises.push(null)
+      continue
+    }
+
     const id = b4a.toString(ref.oplog.key, 'hex')
     // only seeds the node - reapplication re-resolves from the witness
     const weight = ref.oplog.weight
@@ -215,7 +212,7 @@ async function inflate(auto, reorder) {
       sessions.set(id, core)
     }
 
-    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i))
+    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i, readOpts))
   }
 
   try {
@@ -225,11 +222,27 @@ async function inflate(auto, reorder) {
   }
 }
 
-async function getOplogBatch(undo, core, length, weight, index) {
+async function getOplogBatch(undo, core, length, weight, index, readOpts = null) {
   const seq = length - 1
-  const block = await core.get(seq)
+
+  const block = await core.get(seq, readOpts)
+  if (block === null) return null
+
   const oplog = encoding.decodeOplog(block)
   const head = createNode(core, length, weight, oplog)
+  const legacy = oplog.version <= LEGACY_OPLOG_VERSION
+
+  // if this is legacy the start pointer is always 0, fix
+  if (legacy) {
+    for (let i = length - 1; i > 0; i--) {
+      const block = await core.get(i - 1, readOpts)
+      if (block === null) return null // wait: false in replay helper
+
+      const oplog = encoding.decodeOplog(block)
+      if (oplog.batch.end === 0) break
+      head.batch.start++
+    }
+  }
 
   const batch = []
   const result = { undo, batch, index }
@@ -239,18 +252,26 @@ async function getOplogBatch(undo, core, length, weight, index) {
   const end = seq + head.batch.end // skip head, we have it
 
   for (let i = start; i < end; i++) {
-    remaining.push(core.get(i))
+    remaining.push(core.get(i, readOpts))
   }
 
   if (remaining.length) {
     const blocks = await Promise.all(remaining)
     for (let i = 0; i < blocks.length; i++) {
+      // keep the hole so the gap in the batch stays visible
+      if (blocks[i] === null) {
+        batch.push(null)
+        continue
+      }
+
       const oplog = encoding.decodeOplog(blocks[i])
       batch.push(createNode(core, i + start + 1, weight, oplog))
     }
   }
 
   batch.push(head)
+  finalizeNodeBatch(batch)
+
   return result
 }
 
@@ -259,7 +280,9 @@ function getLocalBatch(undo, writer, length, weight, index) {
 
   if (length > writer.core.length) {
     if (!writer.pending) {
-      throw new Error(`No local nodes: looking for ${length}, found ${writer.core.length}`)
+      throw new Error(
+        `No local nodes: looking for ${length}, found ${writer.core.length}@${writer.core.id}`
+      )
     }
 
     let i = 0
@@ -269,9 +292,10 @@ function getLocalBatch(undo, writer, length, weight, index) {
     }
 
     const head = writer.pending[i]
+    const headBatch = head.batch || { start: 0, end: 0 }
 
-    const start = i - head.batch.start
-    const end = i + head.batch.end + 1 // include head
+    const start = i - headBatch.start
+    const end = i + headBatch.end + 1 // include head
 
     const batch = writer.pending.slice(start, end)
     return { undo, batch, index }
@@ -280,6 +304,19 @@ function getLocalBatch(undo, writer, length, weight, index) {
   return getOplogBatch(undo, writer.core, length, weight, index)
 }
 
+// tolerates legacy nodes instead of asserting
+async function* oplogIterator(bee) {
+  try {
+    for await (const node of bee.createChangesStream({ wait: false })) {
+      const legacy = isLegacyNode(node)
+      const refs = getOplogCompat(node)
+      if (refs.length) yield { node, refs, legacy }
+    }
+  } catch {
+    yield null // signal data not available
+  }
+}
+
 function getOplog(data) {
   const result = { ref: null, info: null }
   for (const { keys } of data.batch) {
@@ -305,6 +342,35 @@ function getOplog(data) {
   asserts.bail('Bad system node')
 }
 
+// legacy nodes carry no isOplog flag so we cannot tell which writer update is
+// the oplog to fetch - yield every candidate until we have a better signal
+function getOplogCompat(data) {
+  const undo = data.tail || EMPTY_HEAD
+  const oplogs = []
+  const candidates = []
+
+  for (const { keys } of data.batch) {
+    for (const k of keys) {
+      if (k.key[0] !== 1) continue
+
+      let value = null
+      try {
+        value = encoding.decodeSystemWriter(k.key, k.value)
+      } catch {
+        continue
+      }
+
+      // writer was just added, nothing to fetch
+      if (!value.length) continue
+
+      if (value.isOplog) oplogs.push({ undo, oplog: value })
+      else candidates.push({ undo, oplog: value })
+    }
+  }
+
+  return oplogs.length ? oplogs : candidates
+}
+
 function isLinking(node, link) {
   if (node.length >= link.length && b4a.equals(node.key, link.key)) {
     return true
@@ -335,11 +401,3 @@ function isLegacyNode(data) {
   }
   return false
 }
-
-function sameNode(a, b) {
-  return b4a.equals(a.key, b.key) && a.length === b.length
-}
-
-function getBatch(b) {
-  return b.batch
-}

package/lib/trusted.js

@@ -0,0 +1,44 @@
+const b4a = require('b4a')
+
+class TrustedPeers {
+  constructor(hooks) {
+    this.trusted = new Map()
+    this.pending = new Map()
+    this.hooks = hooks
+  }
+
+  async isTrusted(key, reference) {
+    const hex = b4a.toString(key, 'hex')
+    if (this.trusted.get(hex)) return true
+
+    let promise = this.pending.get(hex)
+    if (promise) return promise
+
+    promise = this.hooks.isTrusted
+      ? this.hooks.isTrusted(key, reference)
+      : Promise.resolve(!this.hooks.mostRecentTrusted)
+    this.pending.set(hex, promise)
+
+    const trusted = await promise
+    if (trusted) this.trusted.set(hex, trusted)
+
+    if (this.pending.get(hex) === promise) this.pending.delete(hex)
+    return trusted
+  }
+
+  read(trusted) {
+    return trusted || null
+  }
+
+  mostRecentTrusted(target, reference) {
+    if (!this.hooks.mostRecentTrusted) return Promise.resolve(null)
+    return this.hooks.mostRecentTrusted(target, reference)
+  }
+
+  clear() {
+    this.pending.clear()
+    this.trusted.clear()
+  }
+}
+
+module.exports = TrustedPeers

package/lib/updates.js

@@ -1,5 +1,3 @@
-const b4a = require('b4a')
-
 module.exports = class UpdateChanges {
   constructor(auto) {
     this.auto = auto
@@ -21,57 +19,22 @@ module.exports = class UpdateChanges {
         flushes: this.auto.system.flushes,
         from: this.tracking[k].head(),
         to: null,
-        shared: null
+        incremental: false
       })
     }
   }
 
   finalise() {
-    const floor = this.auto.system.shared
-
     // flush count at `to` - safe to read live, state is frozen for the hook
     const flushes = this.auto.system.flushes
 
     for (const k in this.tracking) {
       const tracked = this.byName.get(k)
-      const startFlushes = tracked.flushes
 
       tracked.to = this.tracking[k].head()
       tracked.flushes = flushes
-
-      if (floor && floor.flushes < startFlushes) {
-        tracked.shared = k === 'view' ? floor.view : floor.system
-      } else {
-        tracked.shared = tracked.from
-      }
     }
 
     return this.byName
   }
-
-  static from(shared, current) {
-    if (
-      shared.flushes === current.flushes &&
-      shared.system.length === current.system.length &&
-      (shared.system.key === current.system.key ||
-        (shared.system.key &&
-          current.system.key &&
-          b4a.equals(shared.system.key, current.system.key)))
-    ) {
-      return null
-    }
-
-    const byName = new Map()
-    const flushes = current.flushes
-
-    byName.set('view', { from: shared.view, shared: shared.view, to: current.view, flushes })
-    byName.set('system', {
-      from: shared.system,
-      shared: shared.system,
-      to: current.system,
-      flushes
-    })
-
-    return byName
-  }
 }

package/lib/witness.js

@@ -1,94 +1,63 @@
 const b4a = require('b4a')
-const c = require('compact-encoding')
-const crypto = require('hypercore-crypto')
-const Hypercore = require('hypercore')
-const { NS_WITNESS } = require('./constants.js')
+const encoding = require('./encoding.js')
 
 exports.currentWeight = currentWeight
 exports.resolveWeight = resolveWeight
-exports.verifyWitness = verifyWitness
-exports.attestationMessage = attestationMessage
-exports.signAttestation = signAttestation
+exports.verifyClaim = verifyClaim
 
-// resolved current weight of a writer
 function currentWeight(rec) {
   if (!rec) return 0
   if (rec.maxWeight > 0 && rec.isGenesis) return rec.maxWeight
   return rec.weight
 }
 
-// resolve witness weight to sort weight according to current system
+function seedWeight(auto, node) {
+  if (node.length === 1 && b4a.equals(node.key, auto.key)) return auto.bootstrapWeight
+  return 0
+}
+
 async function resolveWeight(auto, node) {
   const rec = await auto.system.get(node.key)
-  const prev = currentWeight(rec)
+  const prev = rec ? currentWeight(rec) : seedWeight(auto, node)
 
   const witness = node.witness
-  if (!witness || witness.weight <= prev) return prev
+  if (!witness) return prev
 
-  const limit = rec ? rec.maxWeight : 0
-  if (witness.weight > limit) return prev
+  const { weight, link } = witness
+  if (weight <= prev) return prev
+  if (!(await verifyClaim(auto, node.key, weight, link))) return prev
 
-  // a node never influences its own sort key
-  if (b4a.equals(witness.backer.key, node.key)) return prev
+  return weight
+}
 
-  const backer = await auto.system.get(witness.backer.key)
-  if (!backer || backer.length < witness.backer.length) return prev
+async function verifyClaim(auto, key, weight, link) {
+  if (!link || !link.length) return false
 
-  if (!verifyWitness(node)) return prev
+  const anchor = await readOplogNode(auto, link.key, link.length)
+  const entry = anchor.approvals ? anchor.approvals.find((a) => b4a.equals(a.key, key)) : null
+  if (!entry || entry.weight < weight) return false
 
-  return Math.max(prev, witness.weight)
+  return approverQualified(auto, link.key, entry.weight)
 }
 
-// a pure function of the node's own bytes - no core session, no store
-// lookup, no IO at all, so it returns the same verdict on every peer at
-// every point in time, including peers that fast-forwarded past the
-// backer's history and never ingested its blocks. the inlined manifest is
-// bound to the cited backer key (a v1+ core key is the manifest hash), the
-// signer public key read out of it, and the signature checked against that
-function verifyWitness(node) {
-  const witness = node.witness
-  if (!witness || !witness.backer || witness.weight === 0) return false
-  if (b4a.equals(witness.backer.key, node.key)) return false
-
-  const publicKey = signerFromRawManifest(witness.backer.key, witness.backer.manifest)
-  if (!publicKey) return false
+// the approval sits in the causal past of any citation of it, so the approver's
+// record is lower-bounded by the standing it held when it approved - no need to
+// walk its own witness chain back to genesis
+async function approverQualified(auto, key, weight) {
+  if (b4a.equals(key, auto.key)) return auto.bootstrapWeight >= weight
 
-  const message = attestationMessage(
-    witness.backer.key,
-    node.key,
-    witness.backer.length,
-    witness.weight
-  )
-  return crypto.verify(message, witness.backer.signature, publicKey)
+  const rec = await auto.system.get(key)
+  return currentWeight(rec) >= weight
 }
 
-function signerFromRawManifest(backerKey, raw) {
-  if (!raw || !raw.length) return null
+async function readOplogNode(auto, key, length) {
+  const core = auto.openCore(key)
 
-  let manifest
   try {
-    // throws on malformed bytes AND when the manifest does not match the key
-    manifest = Hypercore.parseManifest(raw, backerKey)
-  } catch {
-    return null
+    await core.ready()
+    const block = await core.get(length - 1)
+    return encoding.decodeOplog(block)
+  } finally {
+    await core.close()
   }
-
-  // single-signer backers only: an attestation is one writer's testimony
-  if (!manifest || !manifest.signers || manifest.signers.length !== 1) return null
-
-  return manifest.signers[0].publicKey
-}
-
-function attestationMessage(backerKey, key, length, weight) {
-  return b4a.concat([
-    NS_WITNESS,
-    backerKey,
-    key,
-    c.encode(c.uint, length),
-    c.encode(c.uint, weight)
-  ])
-}
-
-function signAttestation(secretKey, backerKey, key, length, weight) {
-  return crypto.sign(attestationMessage(backerKey, key, length, weight), secretKey)
 }