autobee 2.0.0-rc.9 → 2.0.0

package/lib/system.js

@@ -1,4 +1,5 @@
 const Hyperbee = require('hyperbee2')
+const Promotions = require('./promotions.js')
 const b4a = require('b4a')
 
 const { AUTOBEE_VERSION } = require('./constants')
@@ -43,31 +44,39 @@ module.exports = class Systembee {
     this.version = 0
     this.timestamp = 0
     this.flushes = 0
-    this.indexers = null // legacy
     this.updates = new Map()
     this.writers = new Map()
-    this.shared = null
     this.encrypted = opts.encrypted === true
+    this.promotions = new Promotions(this.bee)
   }
 
-  // clamps weight to max(requested, current). order-independent, so
-  // concurrent races converge and honest weights never decrease
-  async addWriter(key, { length = 0, weight = 1, isGenesis = false } = {}) {
+  async addWriter(
+    key,
+    { length = 0, weight = 1, isGenesis = false, coord = null, carrier = -1, anchor = false } = {}
+  ) {
     const info = await this.get(key, { unflushed: true })
     if (length === 0) {
       length = info ? info.length : 0
     }
 
+    const effective = coord && carrier < weight ? Math.max(carrier, 0) : weight
+    const hint =
+      anchor && coord ? { key: coord.key, length: coord.length, weight: effective } : null
+    const pending = coord && !isGenesis ? weight : -1
+
     const current = info ? info.maxWeight : 0
-    if (weight <= current) {
-      if (info && !info.isRemoved && length <= info.length) return
+    if (effective <= current) {
+      // isAdded is false only when nothing has recorded the add yet - skipping
+      // the update then leaves the writer undiscoverable, never announced
+      const added = info && info.isAdded !== false
+      if (added && !info.isRemoved && length <= info.length && !hint && pending === -1) return
 
-      this.update(key, length, -1, -1, false, true, false, false, false, -1, false)
+      this.update(key, length, -1, -1, false, true, false, -1, hint, pending)
       return
     }
 
-    const w = isGenesis ? weight : -1
-    this.update(key, length, w, weight, isGenesis, true, false, false, false, -1, true)
+    const w = isGenesis ? effective : -1
+    this.update(key, length, w, effective, isGenesis, true, false, -1, hint, pending)
   }
 
   async ackWriter(key, { length = 0 } = {}) {
@@ -75,7 +84,7 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: false })
       length = info ? info.length : 0
     }
-    this.update(key, length, -1, -1, false, false, false, true, false, -1, false)
+    this.update(key, length, -1, -1, false, false, false, -1, null)
   }
 
   async removeWriter(key, { length = 0 } = {}) {
@@ -83,17 +92,24 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: true })
       length = info ? info.length : 0
     }
-    this.update(key, length, -1, 0, false, false, true, false, false, -1, false)
+    this.update(key, length, -1, 0, false, false, true, -1, null)
   }
 
-  addAnchor(key, { length = 1 } = {}) {
-    this.update(key, length, -1, -1, false, true, false, false, true, -1, false)
+  addAnchor(key) {
+    this.update(key, 0, -1, -1, false, true, false, -1, null)
   }
 
   isGenesis() {
     return this.bee.head() === null || this.bee.head().length === 0
   }
 
+  isHead(key) {
+    for (const head of this.heads) {
+      if (b4a.equals(head.key, key)) return true
+    }
+    return false
+  }
+
   async boot(view, { timeout } = {}) {
     await this.bee.ready()
     this.bee.move(view)
@@ -115,60 +131,6 @@ module.exports = class Systembee {
     }
   }
 
-  async commonAncestor(head) {
-    const snap = this.bee.checkout(head)
-
-    const ca = infoIterator(this.bee)
-    const cb = infoIterator(snap)
-
-    let l = await next(ca)
-    let r = await next(cb)
-
-    while (l && r) {
-      if (l.legacy && r.legacy) {
-        // legacy nodes just compare head
-        if (sameHead(l.node.head, r.node.head)) {
-          return {
-            flushes: l.info.flushes,
-            system: l.node.head,
-            view: l.info.view || EMPTY_HEAD
-          }
-        }
-
-        // different legacy cores never converge - genuinely unrelated history
-        if (!b4a.equals(l.node.head.key, r.node.head.key)) return null
-
-        if (l.node.head.length >= r.node.head.length) l = await next(ca)
-        else r = await next(cb)
-        continue
-      }
-
-      if (l.legacy !== r.legacy) {
-        // current-format history sits above the legacy band: descend it
-        if (!l.legacy) l = await next(ca)
-        else r = await next(cb)
-        continue
-      }
-
-      if (l.info.flushes === r.info.flushes && sameHead(l.info.view, r.info.view)) {
-        return {
-          flushes: l.info.flushes,
-          system: l.node.head,
-          view: l.info.view || EMPTY_HEAD
-        }
-      }
-
-      const lf = l.info.flushes
-      const rf = r.info.flushes
-
-      if (lf >= rf) l = await next(ca)
-      if (rf >= lf) r = await next(cb)
-    }
-
-    // no common ancestor
-    return null
-  }
-
   async reset({ timeout } = {}) {
     const info = await this.getInfo({ timeout })
 
@@ -177,10 +139,9 @@ module.exports = class Systembee {
     this.heads = info ? info.heads : []
     this.timestamp = info ? info.timestamp : 0
     this.flushes = info ? info.flushes : 0
-    this.indexers = info ? info.indexers : null
-    this.shared = null
     this.updates.clear()
     this.writers.clear()
+    this.promotions.reset(info)
 
     if (this.version > AUTOBEE_VERSION) {
       throw new Error('Autobee signals newer version than locally supported')
@@ -200,16 +161,8 @@ module.exports = class Systembee {
   async undo(head) {
     this.bee.move(head)
 
-    const shared = this.shared
     await this.reset()
 
-    // deepest rewind since last mark - everything at or below it is untouched
-    if (shared === null || this.flushes < shared.flushes) {
-      this.shared = { flushes: this.flushes, view: this.view || EMPTY_HEAD, system: head }
-    } else if (shared) {
-      this.shared = shared
-    }
-
     return this.view || EMPTY_HEAD
   }
 
@@ -238,19 +191,7 @@ module.exports = class Systembee {
 
     // resolved sort weight, re-stamped on every reapplication - this is what
     // topo reads back out of the changes stream
-    this.update(
-      node.key,
-      node.length,
-      node.weight,
-      -1,
-      false,
-      false,
-      false,
-      false,
-      false,
-      node.timestamp,
-      false
-    )
+    this.update(node.key, node.length, node.weight, -1, false, false, false, node.timestamp, null)
   }
 
   async canApply(key, optimistic) {
@@ -269,8 +210,11 @@ module.exports = class Systembee {
     return new SystemSnapshot(bee)
   }
 
-  async get(key, { unflushed = false, timeout } = {}) {
-    const node = await this.bee.get(encoding.encodeSystemWriterKey(key), { timeout })
+  async get(key, { unflushed = false, timeout, activeRequests } = {}) {
+    const node = await this.bee.get(encoding.encodeSystemWriterKey(key), {
+      timeout,
+      activeRequests
+    })
     const info = node !== null ? encoding.decodeSystemWriter(node.key, node.value) : null
     if (!unflushed) return info
 
@@ -288,16 +232,12 @@ module.exports = class Systembee {
         isGenesis: upd.maxWeight !== -1 && !!upd.isGenesis,
         length: upd.length,
         clock: 0,
-        isAdded: upd.isAdded,
-        isAcked: upd.isAcked,
-        isAnchor: upd.isAnchor
+        isAdded: upd.isAdded
       }
     }
 
     if (upd.isAdded) info.isAdded = true
     if (upd.isRemoved) info.isRemoved = true
-    if (upd.isAcked) info.isAcked = true
-    if (upd.isAnchor) info.isAnchor = true
     if (upd.maxWeight !== -1) {
       info.isGenesis = upd.isGenesis
       info.maxWeight = upd.maxWeight
@@ -338,17 +278,16 @@ module.exports = class Systembee {
     isGenesis,
     isAdded,
     isRemoved,
-    isAcked,
-    isAnchor,
     timestamp,
-    needsWitness
+    grant,
+    pending = -1
   ) {
     const id = b4a.toString(key, 'hex')
 
     if (isAdded) {
-      this.writers.set(id, { key, added: true, isAnchor: !!isAnchor })
+      this.writers.set(id, { key, added: true })
     } else if (isRemoved) {
-      this.writers.set(id, { key, added: false, isAnchor: !!isAnchor })
+      this.writers.set(id, { key, added: false })
     }
 
     let upd = this.updates.get(id)
@@ -362,10 +301,9 @@ module.exports = class Systembee {
         isGenesis: false,
         isRemoved: false,
         isAdded: false,
-        isAcked: false,
-        isAnchor: false,
-        timestamp: -1,
-        needsWitness
+        timestamp: 0,
+        grant: null,
+        pending: -1
       }
       this.updates.set(id, upd)
     }
@@ -376,7 +314,6 @@ module.exports = class Systembee {
       upd.maxWeight = maxWeight
       upd.isGenesis = isGenesis
     }
-    if (isAnchor) upd.isAnchor = true
     if (isAdded) {
       upd.isAdded = true
       upd.isRemoved = false
@@ -385,18 +322,16 @@ module.exports = class Systembee {
       upd.isRemoved = true
       upd.isAdded = false
     }
-    if (isAcked) {
-      upd.isAcked = true
-    }
     if (length > upd.length) {
       upd.length = length
     }
 
-    if (timestamp !== -1 && upd.timestamp === -1) {
+    if (timestamp > -1) {
       upd.timestamp = timestamp
     }
-    if (needsWitness && !upd.needsWitness) {
-      upd.needsWitness = true
+    if (pending > upd.pending) upd.pending = pending
+    if (grant && (upd.grant === null || grant.weight > upd.grant.weight)) {
+      upd.grant = grant
     }
   }
 
@@ -411,55 +346,54 @@ module.exports = class Systembee {
     const isOplog = b4a.equals(upd.key, oplog)
 
     const record = {
-      version: 4,
+      version: 5,
       key: upd.key,
       isRemoved: upd.isRemoved ? true : upd.isAdded ? false : v ? v.isRemoved : false,
       isOplog,
+      isGenesis: upd.maxWeight !== -1 ? !!upd.isGenesis : v ? v.isGenesis : false,
       weight: upd.weight !== -1 ? upd.weight : prevWeight,
       maxWeight: upd.maxWeight !== -1 ? upd.maxWeight : prevMaxWeight,
-      isGenesis: upd.maxWeight !== -1 ? !!upd.isGenesis : v ? v.isGenesis : false,
       length: Math.max(upd.length, v ? v.length : 0),
-      clock: this.flushes,
-      isAnchor: upd.isAnchor || (v ? v.isAnchor : false),
-      timestamp: isOplog ? upd.timestamp : -1
+      timestamp: isOplog ? upd.timestamp : 0
     }
 
     return {
       key: k,
       value: encoding.encodeSystemWriter(record),
       add: null,
-      remove: record.isRemoved ? encoding.encodeSystemIndexerKey(upd.key) : null,
+      remove: null,
       maxWeight: record.maxWeight
     }
   }
 
   async flush(batch, bee) {
-    if (batch.length === 0) return { witnessed: [], changed: [] }
+    if (batch.length === 0) return []
 
     const oplog = batch[batch.length - 1].key
 
     const updates = [...this.updates.values()]
-    const results = await Promise.all(updates.map((upd) => this._updateWriter(upd, oplog)))
-
-    const witnessed = []
-    for (let i = 0; i < updates.length; i++) {
-      if (updates[i].needsWitness) {
-        witnessed.push({ key: updates[i].key, weight: results[i].maxWeight })
-      }
-    }
+    const results = await Promise.all(updates.map((upd) => this._flushUpdate(upd, oplog)))
+    const pendings = await this.promotions.flushPending(updates)
 
     const w = this.bee.write()
 
-    // migrate
-    if (this.indexers) {
-      for (const { key } of this.indexers) w.tryPut(encoding.encodeSystemIndexerKey(key), EMPTY)
-      this.indexers = null
+    for (const { writer } of results) {
+      w.tryPut(writer.key, writer.value)
+      if (writer.add) w.tryPut(writer.add, EMPTY)
+      if (writer.remove) w.tryDelete(writer.remove)
     }
 
-    for (const { key, value, add, remove } of results) {
-      w.tryPut(key, value)
-      if (add) w.tryPut(add, EMPTY)
-      if (remove) w.tryDelete(remove)
+    for (const { grant } of results) {
+      if (!grant) continue
+      if (grant.remove) w.tryDelete(grant.key)
+      else w.tryPut(grant.key, grant.value)
+    }
+
+    if (pendings) {
+      for (const p of pendings) {
+        if (p.remove) w.tryDelete(p.key)
+        else w.tryPut(p.key, p.value)
+      }
     }
 
     this.flushes++
@@ -470,11 +404,15 @@ module.exports = class Systembee {
       heads: this.heads,
       timestamp: this.timestamp,
       flushes: this.flushes,
-      indexers: null // legacy
+      indexers: null, // legacy
+      pending: this.promotions.digest
     }
 
     w.tryPut(INFO_KEY, encoding.encodeSystemInfo(info))
 
+    this.version = info.version
+    this.view = info.view
+
     await w.flush()
 
     // Can enable if needed during debuggin
@@ -482,64 +420,43 @@ module.exports = class Systembee {
 
     this.updates.clear()
 
-    if (this.writers.size === 0) return { witnessed, changed: [] }
+    if (this.writers.size === 0) return []
 
     const changes = [...this.writers.values()]
 
     this.writers.clear()
 
-    return {
-      witnessed,
-      changed: changes
-    }
+    return changes
   }
-}
 
-function getInfo(data) {
-  for (const { keys } of data.batch) {
-    for (const k of keys) {
-      const prefix = k.key[0]
+  async _flushUpdate(upd, oplog) {
+    const [writer, grant] = await Promise.all([
+      this._updateWriter(upd, oplog),
+      this.promotions.updateHint(upd)
+    ])
 
-      if (prefix === 0) {
-        try {
-          return encoding.decodeSystemInfo(k.value)
-        } catch {
-          return null
-        }
-      }
-    }
+    return { writer, grant }
   }
 
-  return null
-}
-
-function sameHead(a, b) {
-  if (!a || !b || a.length !== b.length) return false
-  if (!a.key || !b.key) return a.key === b.key
-  return b4a.equals(a.key, b.key)
-}
+  pendingPromotion(key, opts) {
+    return this.promotions.pending(key, opts)
+  }
 
-async function* infoIterator(bee) {
-  try {
-    for await (const node of bee.createChangesStream()) {
-      const legacy = topo.isLegacyNode(node)
-      const info = legacy ? getLegacyInfo(node) : getInfo(node)
-      if (info) yield { node, info, legacy }
-    }
-  } catch {
-    // fall through
+  listPendingPromotions(opts) {
+    return this.promotions.list(opts)
   }
-}
 
-async function next(it) {
-  const res = await it.next()
-  return res.done ? null : res.value
+  grantHint(key, opts) {
+    return this.promotions.hint(key, opts)
+  }
 }
 
-function getLegacyInfo(data) {
+function getInfo(data) {
   for (const { keys } of data.batch) {
     for (const k of keys) {
-      if (b4a.equals(k.key, INFO_LEGACY_KEY)) {
+      const prefix = k.key[0]
+
+      if (prefix === 0) {
         try {
           return encoding.decodeSystemInfo(k.value)
         } catch {
@@ -548,5 +465,6 @@ function getLegacyInfo(data) {
       }
     }
   }
+
   return null
 }