autobee 2.0.0-rc.2 → 2.0.0-rc.21

package/lib/apply-calls.js

@@ -1,4 +1,5 @@
 const ID = require('hypercore-id-encoding')
+const b4a = require('b4a')
 
 class ApplyCalls {
   constructor(auto) {
@@ -39,6 +40,10 @@ class ApplyCalls {
 
   ackWriter(key) {
     if (typeof key === 'string') key = ID.decode(key)
+    // acking an optimistic node means we owe it a linking node
+    if (this.applying && this.applying[0].optimistic && !b4a.equals(key, this.auto.local.key)) {
+      this.auto._ackRequired = true
+    }
     return this.auto.system.ackWriter(key)
   }
 

package/lib/apply-view.js

@@ -0,0 +1,25 @@
+module.exports = class ApplyView {
+  constructor(bee, auto) {
+    this.bee = bee
+    this.auto = auto
+    this.view = ApplyView.open(bee, auto)
+  }
+
+  static open(bee, auto) {
+    return auto._handlers.open ? auto._handlers.open(bee, auto) : bee
+  }
+
+  static close(view, auto) {
+    if (!auto._handlers.close) return Promise.resolve()
+    return auto._handlers.close(view)
+  }
+
+  apply(userBatch) {
+    return this.auto._handlers.apply(userBatch, this.view, this.auto._host)
+  }
+
+  async close() {
+    await ApplyView.close(this.view, this.auto)
+    await this.bee.close()
+  }
+}

package/lib/boot.js

@@ -201,6 +201,8 @@ async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
     mappedViews.set(name, v)
   }
 
+  await core.close()
+
   return {
     encryptionKey,
     system,
@@ -242,6 +244,7 @@ async function getCatchupHeads(core) {
     if (entry.key[0] !== 0x01) continue
 
     const { key, length } = encoding.decodeSystemWriter(entry.key, entry.value)
+    if (!length) continue
     nodes.push({ key, length })
   }
 
@@ -254,7 +257,9 @@ async function getWriterBatch(store, head, key, encryptionKey, nodes = []) {
   const core = store.get(head.key)
   await core.ready()
 
-  await core.setEncryption(AutobeeEncryption.getWriterEncryption(key, encryptionKey))
+  if (encryptionKey) {
+    await core.setEncryption(AutobeeEncryption.getWriterEncryption(key, encryptionKey))
+  }
 
   let seq = head.length - 1
   const block = await core.get(seq--, { wait: false })

package/lib/encoding.js

@@ -12,6 +12,7 @@ const SystemInfo = getEncoding('@autobee/system-info')
 const SystemWriter = getEncoding('@autobee/system-writer')
 const ManifestData = getEncoding('@autobee/manifest-data')
 const AutobaseBootRecord = getEncoding('@autobase-compat/boot-record')
+const MigratedHead = getEncoding('@autobee/migrated-head')
 
 exports.OPLOG_VERSION = OPLOG_VERSION
 
@@ -21,6 +22,9 @@ exports.ManifestData = ManifestData
 exports.encodeBootRecord = encodeBootRecord
 exports.decodeBootRecord = decodeBootRecord
 
+exports.encodeMigratedHead = encodeMigratedHead
+exports.decodeMigratedHead = decodeMigratedHead
+
 exports.encodePreviousDrain = encodePreviousDrain
 exports.decodePreviousDrain = decodePreviousDrain
 
@@ -39,6 +43,14 @@ function decodeBootRecord(value) {
   return c.decode(Link, value)
 }
 
+function encodeMigratedHead(m) {
+  return c.encode(MigratedHead, m)
+}
+
+function decodeMigratedHead(value) {
+  return c.decode(MigratedHead, value)
+}
+
 function encodePreviousDrain(d) {
   return c.encode(c.uint, d)
 }

package/lib/{reboot.js → fast-forward.js}

@@ -1,5 +1,6 @@
 const safetyCatch = require('safety-catch')
 const b4a = require('b4a')
+const Hypercore = require('hypercore')
 const c = require('compact-encoding')
 const crypto = require('hypercore-crypto')
 
@@ -18,13 +19,24 @@ const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
 
 const EMPTY = b4a.alloc(0)
 const DEFAULT_OP_TIMEOUT = 5_000
+const MIN_FF_GAP = 32
 
-module.exports = class Reboot {
+// views are only stamped on the last node of a flush, so a head may sit above
+// the most recent one - how far back we look for it
+const MAX_FLUSH_SEARCH = 64
+
+module.exports = class FastForward {
   constructor(auto, head, tip, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
     this.auto = auto
+
+    this.activeRequests = []
+
+    // 'reboot' is a storage namespace: renaming it would derive a different
+    // scratch core on disk, so it stays as is
     this.system = new System(auto.store.namespace('reboot'), null, {
       getEncryptionProvider: this.auto.getSystemEncryption,
-      encrypted: this.encrypted
+      encrypted: this.encrypted,
+      activeRequests: this.activeRequests
     })
 
     // length -1 marks an uninitialised head: we know the key but not the length
@@ -37,6 +49,164 @@ module.exports = class Reboot {
     this.cores = []
   }
 
+  static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
+
+  static async fromHead(
+    auto,
+    head,
+    trusted,
+    { force = false, timeout = 0, condition = null, reference = null } = {}
+  ) {
+    const conservative = !force && auto._conservativeFF
+    const timeoutOpts = timeout ? { timeout } : null
+
+    const oplog = await FastForward.flushHead(auto, head, {
+      conservative,
+      ...timeoutOpts
+    })
+    if (!oplog) return null
+
+    const verified = trusted ? await FastForward.flushHead(auto, trusted, timeoutOpts) : oplog
+    if (!verified) return null
+
+    // legacy nodes from non-indexers have no system info to fast-forward from
+    if (!oplog.op.views || !verified.op.views) return null
+
+    if (!force && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
+      return null
+    }
+
+    if (condition !== null && !(await FastForward.accepts(auto, verified, condition, reference))) {
+      return null
+    }
+
+    const tip = {
+      system: batchToHead(oplog.op.views.system),
+      verified: {
+        op: { key: verified.key, length: verified.length },
+        flushes: verified.op.views.flushes
+      }
+    }
+
+    return new FastForward(auto, batchToHead(verified.op.views.system), tip)
+  }
+
+  static async fromHeads(
+    auto,
+    heads,
+    { force = false, timeout = 0, condition = null, reference = null } = {}
+  ) {
+    if (reference === null) reference = auto._workingView.view
+
+    const promises = []
+
+    for (const head of heads) {
+      if (head.length === 0) continue
+      promises.push(FastForward.flushHead(auto, head, timeout ? { timeout } : null))
+    }
+
+    const ops = await Promise.all(promises)
+    if (auto.fastForwarding || auto.fastForwardTo) return null
+
+    const trust = await Promise.all(
+      ops.map((res) => {
+        return res === null ? false : auto.trusted.isTrusted(res.key, reference)
+      })
+    )
+    if (auto.fastForwarding || auto.fastForwardTo) return null
+
+    const candidates = []
+
+    let bestTrusted = null
+    let bestTrustedFlushes = -1
+
+    for (let i = 0; i < ops.length; i++) {
+      const res = ops[i]
+      if (res === null || !res.op.views) continue
+      if (!force && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
+
+      if (!trust[i]) {
+        candidates.push(res)
+        continue
+      }
+
+      if (res.op.views.flushes > bestTrustedFlushes) {
+        bestTrustedFlushes = res.op.views.flushes
+        bestTrusted = res
+      }
+    }
+
+    if (bestTrusted !== null) {
+      return FastForward.fromHead(auto, bestTrusted, null, {
+        force,
+        timeout,
+        condition,
+        reference
+      })
+    }
+
+    candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
+
+    for (const res of candidates) {
+      const trusted = await FastForward.mostRecentTrusted(auto, res, reference)
+      if (trusted === null) continue
+      if (auto.fastForwarding || auto.fastForwardTo) return null
+
+      const ff = await FastForward.fromHead(auto, res, trusted, {
+        force,
+        timeout,
+        condition,
+        reference
+      })
+      if (ff !== null) return ff
+    }
+
+    return null
+  }
+
+  static async flushHead(auto, head, opts) {
+    const core = auto.openCore(head.key)
+
+    try {
+      let length = head.length
+
+      for (let i = 0; i < MAX_FLUSH_SEARCH; i++) {
+        const oplog = await auto.readOplog(core, length, opts)
+        if (oplog === null) return null
+        if (oplog.op.views) return oplog
+
+        length = oplog.length - 1
+        if (length <= 0) return null
+      }
+
+      return null
+    } finally {
+      await core.close()
+    }
+  }
+
+  static async accepts(auto, oplog, condition, reference) {
+    const opened = auto.openViewAt(oplog)
+    if (opened === null) return false
+
+    try {
+      return !!(await condition(opened.view, reference))
+    } finally {
+      await opened.close()
+    }
+  }
+
+  static async mostRecentTrusted(auto, head, reference) {
+    const opened = auto.openViewAt(head)
+    if (opened === null) return null
+
+    try {
+      return (await auto.trusted.mostRecentTrusted(opened.view, reference)) || null
+    } finally {
+      await opened.close()
+    }
+  }
+
   async run() {
     try {
       if (!this.running) this.running = this._run()
@@ -64,8 +234,15 @@ module.exports = class Reboot {
 
     // a legacy system needs migrating rather than a plain fast-forward
     if (this.system.version <= LEGACY_AUTOBASE_VERSION) {
-      // already migrated, don't migrate again
-      if (this.auto.system.version > LEGACY_AUTOBASE_VERSION) return null
+      const migrated = this.auto._migratedHead && this.auto._migratedHead.system
+      if (
+        migrated &&
+        b4a.equals(migrated.key, this.head.key) &&
+        migrated.length >= this.head.length
+      ) {
+        return null
+      }
+
       return this._migrate()
     }
 
@@ -77,7 +254,12 @@ module.exports = class Reboot {
     const view = this.auto.store.get({ key: this.system.view.key, active: true })
     this.cores.push(view)
 
-    promises.push(view.get(this.system.view.length - 1, { timeout: this.timeout }))
+    promises.push(
+      view.get(this.system.view.length - 1, {
+        timeout: this.timeout,
+        activeRequests: this.activeRequests
+      })
+    )
 
     for (const head of this.system.heads) {
       promises.push(this.system.get(head.key, { timeout: this.timeout }))
@@ -86,23 +268,12 @@ module.exports = class Reboot {
     await Promise.all(promises)
     if (this.destroyed) return null
 
-    const shared = await this._commonAncestor()
     if (this.destroyed) return null
 
     return {
       head: this.head,
       tip: this.tip,
-      migrate: null,
-      shared
-    }
-  }
-
-  async _commonAncestor() {
-    try {
-      return await this.auto.system.commonAncestor(this.head)
-    } catch (err) {
-      safetyCatch(err)
-      return null
+      migrate: null
     }
   }
 
@@ -118,7 +289,12 @@ module.exports = class Reboot {
     await core.setEncryption(this.auto.getSystemEncryption())
 
     // Decode hyperbee block using hyperbee2 compat
-    const node = decodeBlock(await core.get(this.head.length - 1, { timeout: this.timeout }))
+    const node = decodeBlock(
+      await core.get(this.head.length - 1, {
+        timeout: this.timeout,
+        activeRequests: this.activeRequests
+      })
+    )
     assert(node.keys.length > 0, 'bad system block')
 
     // Decode system info from hyperbee block using autobee compat
@@ -163,7 +339,12 @@ module.exports = class Reboot {
     await core.ready()
 
     try {
-      if (!core.manifest) await core.get(length - 1, { timeout: this.timeout })
+      if (!core.manifest) {
+        await core.get(length - 1, {
+          timeout: this.timeout,
+          activeRequests: this.activeRequests
+        })
+      }
     } catch {
       return null
     }
@@ -191,13 +372,26 @@ module.exports = class Reboot {
     }
   }
 
+  // cancels every read in flight and fails any that come after
+  clearRequests(err = null) {
+    Hypercore.destroyRequests(this.activeRequests, err)
+  }
+
   async close() {
     this.destroyed = true
+    this.clearRequests()
     if (this.system) await this.system.close()
     for (const core of this.cores) await core.close()
   }
 }
 
+function batchToHead(b) {
+  return {
+    key: b.key,
+    length: b.start + b.length
+  }
+}
+
 function decodeLegacySystemInfo(buffer) {
   const state = { start: 0, end: buffer.length, buffer }
   const version = c.uint.decode(state)

package/lib/system.js

@@ -2,7 +2,6 @@ const Hyperbee = require('hyperbee2')
 const b4a = require('b4a')
 
 const { AUTOBEE_VERSION } = require('./constants')
-const asserts = require('./asserts.js')
 const encoding = require('./encoding.js')
 const topo = require('./topo.js')
 
@@ -13,6 +12,27 @@ const INDEXER_LT = b4a.from([3])
 const EMPTY_HEAD = { length: 0, key: null }
 const EMPTY = b4a.from([])
 
+class SystemSnapshot {
+  constructor(bee) {
+    this.bee = bee
+  }
+
+  async get(key, { timeout } = {}) {
+    const node = await this.bee.get(encoding.encodeSystemWriterKey(key), { timeout })
+    return node !== null ? encoding.decodeSystemWriter(node.key, node.value) : null
+  }
+
+  async has(link) {
+    const node = await this.get(link.key)
+    if (!node || node.length < link.length) return false
+    return true
+  }
+
+  close() {
+    return this.bee.close()
+  }
+}
+
 module.exports = class Systembee {
   constructor(store, name, opts = {}) {
     this.name = name // for debuggin
@@ -26,7 +46,6 @@ module.exports = class Systembee {
     this.indexers = null // legacy
     this.updates = new Map()
     this.writers = new Map()
-    this.shared = null
     this.encrypted = opts.encrypted === true
   }
 
@@ -95,49 +114,6 @@ module.exports = class Systembee {
     }
   }
 
-  async commonAncestor(head) {
-    const info = await this.getInfo()
-
-    const snap = this.bee.checkout(head)
-
-    const hist1 = this.bee.createChangesStream()[Symbol.asyncIterator]()
-    const hist2 = snap.createChangesStream()[Symbol.asyncIterator]()
-
-    let a = await hist1.next()
-    let b = await hist2.next()
-
-    if (!a.value || !b.value) {
-      return { flushes: 0, system: EMPTY_HEAD, view: EMPTY_HEAD }
-    }
-
-    let l = getInfo(a.value)
-    let r = getInfo(b.value)
-
-    while (!a.done || !b.done) {
-      if (l.flushes === r.flushes && sameHead(l.view, r.view)) {
-        return {
-          flushes: l.flushes,
-          system: a.value.head,
-          view: l.view || EMPTY_HEAD
-        }
-      }
-
-      if (l.flushes >= r.flushes) {
-        if (a.done) break
-        a = await hist1.next()
-        if (a.value) l = getInfo(a.value)
-      }
-
-      if (r.flushes >= l.flushes) {
-        if (b.done) break
-        b = await hist2.next()
-        if (b.value) r = getInfo(b.value)
-      }
-    }
-
-    return { flushes: 0, system: EMPTY_HEAD, view: EMPTY_HEAD }
-  }
-
   async reset({ timeout } = {}) {
     const info = await this.getInfo({ timeout })
 
@@ -147,7 +123,6 @@ module.exports = class Systembee {
     this.timestamp = info ? info.timestamp : 0
     this.flushes = info ? info.flushes : 0
     this.indexers = info ? info.indexers : null
-    this.shared = null
     this.updates.clear()
     this.writers.clear()
 
@@ -169,16 +144,8 @@ module.exports = class Systembee {
   async undo(head) {
     this.bee.move(head)
 
-    const shared = this.shared
     await this.reset()
 
-    // deepest rewind since last mark - everything at or below it is untouched
-    if (shared === null || this.flushes < shared.flushes) {
-      this.shared = { flushes: this.flushes, view: this.view || EMPTY_HEAD, system: head }
-    } else if (shared) {
-      this.shared = shared
-    }
-
     return this.view || EMPTY_HEAD
   }
 
@@ -230,6 +197,14 @@ module.exports = class Systembee {
     return info ? !info.isRemoved : !!optimistic
   }
 
+  snapshot(head = null) {
+    const bee =
+      head === null
+        ? this.bee.snapshot()
+        : this.bee.checkout({ length: head.length, key: head.key })
+    return new SystemSnapshot(bee)
+  }
+
   async get(key, { unflushed = false, timeout } = {}) {
     const node = await this.bee.get(encoding.encodeSystemWriterKey(key), { timeout })
     const info = node !== null ? encoding.decodeSystemWriter(node.key, node.value) : null
@@ -436,6 +411,9 @@ module.exports = class Systembee {
 
     w.tryPut(INFO_KEY, encoding.encodeSystemInfo(info))
 
+    this.version = info.version
+    this.view = info.view
+
     await w.flush()
 
     // Can enable if needed during debuggin
@@ -462,16 +440,14 @@ function getInfo(data) {
       const prefix = k.key[0]
 
       if (prefix === 0) {
-        return encoding.decodeSystemInfo(k.value)
+        try {
+          return encoding.decodeSystemInfo(k.value)
+        } catch {
+          return null
+        }
       }
     }
   }
 
-  asserts.bail('Bad system node')
-}
-
-function sameHead(a, b) {
-  if (!a || !b || a.length !== b.length) return false
-  if (!a.key || !b.key) return a.key === b.key
-  return b4a.equals(a.key, b.key)
+  return null
 }