autobee 2.0.0-rc.27 → 2.0.0-rc.29

package/index.js

@@ -853,15 +853,18 @@ module.exports = class Autobee extends ReadyResource {
 
   async _bumpMigratedWriters() {
     const opened = new Set()
+    let updated = false
 
     for (const batch of this._catchupMigratedNodes) {
       await this._processBatch(batch)
+      updated = true
       for (const node of batch) {
         if (node.from) opened.add(node.from)
       }
     }
 
     for (const core of opened) await core.close()
+    return updated
   }
 
   // append a null value node to ack writer
@@ -899,8 +902,9 @@ module.exports = class Autobee extends ReadyResource {
 
   async _bumpPendingWriters() {
     if (this._catchupMigratedNodes !== null) {
-      await this._bumpMigratedWriters()
+      const updated = await this._bumpMigratedWriters()
       this._catchupMigratedNodes = null
+      if (updated) return true
     }
 
     // apply the best next node to keep the prefix stable
@@ -1182,7 +1186,7 @@ module.exports = class Autobee extends ReadyResource {
   // legacy: ungated boot straight onto a system head
   async _bootFromSystem(system) {
     try {
-      const ff = new FastForward(this, system, null, { timeout: FastForward.DEFAULT_TIMEOUT })
+      const ff = new FastForward(this, system, { timeout: FastForward.DEFAULT_TIMEOUT })
       return await this._runFastForward(ff)
     } catch (err) {
       safetyCatch(err)
@@ -1322,7 +1326,7 @@ module.exports = class Autobee extends ReadyResource {
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
-    const { head, tip, migrate } = this.fastForwardTo
+    const { head, migrate } = this.fastForwardTo
 
     const from = this.system.bee.head()
     const to = head
@@ -1362,28 +1366,6 @@ module.exports = class Autobee extends ReadyResource {
     this.stats.fastForwards++
     this.emit('move-to', to, from)
     this.ff.resolve({ to, from })
-
-    // tip is null during boot
-    if (!tip) return
-
-    try {
-      await this._reapply(tip)
-    } catch (err) {
-      throw err
-    }
-  }
-
-  async _reapply({ system, verified }) {
-    const changes = this._hasUpdate ? new UpdateChanges(this) : null
-    if (changes) changes.track()
-
-    const sys = this.system.bee.checkout(system)
-    const t = await topo.rollback(this, sys, verified)
-    await sys.close()
-
-    if (t !== null) await this.applyBacklog(t.tip)
-
-    return this._update(changes)
   }
 
   replay() {

package/lib/boot.js

@@ -1,3 +1,4 @@
+const b4a = require('b4a')
 const encoding = require('./encoding.js')
 const { checkAutobaseMigration } = require('./migrations.js')
 
@@ -94,7 +95,7 @@ module.exports = async function boot(
     await result.bootstrap.setUserData('autobee/local', result.local.key)
     await result.local.setUserData('referrer', result.key)
 
-    result.migration = await checkAutobaseMigration(
+    result.migration = await migrateWithFallback(
       corestore,
       result.local,
       result.bootstrap,
@@ -138,6 +139,23 @@ module.exports = async function boot(
   return result
 }
 
+// the bootstrap pointer names the rotated-to writer, whose legacy boot record
+// can be unusable. the namespace local still carries the pre-rotation record,
+// which points at the last system core that was actually signed
+async function migrateWithFallback(corestore, local, bootstrap, legacyViews) {
+  try {
+    return await checkAutobaseMigration(corestore, local, bootstrap, legacyViews)
+  } catch (err) {
+    const prev = corestore.get({ name: 'local', active: false })
+    await prev.ready()
+
+    if (b4a.equals(prev.key, local.key)) throw err
+    if (!(await prev.getUserData('autobase/boot'))) throw err
+
+    return checkAutobaseMigration(corestore, prev, bootstrap, legacyViews)
+  }
+}
+
 async function getLocalKey(bootstrap) {
   const [legacy, current] = await Promise.all([
     bootstrap.getUserData('autobase/local'),

package/lib/fast-forward.js

@@ -14,7 +14,7 @@ const MIN_FF_GAP = 32
 const MAX_FLUSH_SEARCH = 64
 
 module.exports = class FastForward {
-  constructor(auto, head, tip, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
+  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
     this.auto = auto
 
     this.activeRequests = []
@@ -30,7 +30,6 @@ module.exports = class FastForward {
     // length -1 marks an uninitialised head: we know the key but not the length
     this.head = { key: head.key, length: head.length ?? 0 }
     this.timeout = timeout
-    this.tip = tip || null
     this.destroyed = false
     this.running = null
     this.failed = false
@@ -68,15 +67,7 @@ module.exports = class FastForward {
       return null
     }
 
-    const tip = {
-      system: batchToHead(oplog.op.views.system),
-      verified: {
-        op: { key: verified.key, length: verified.length },
-        flushes: verified.op.views.flushes
-      }
-    }
-
-    return new FastForward(auto, batchToHead(verified.op.views.system), tip)
+    return new FastForward(auto, batchToHead(verified.op.views.system))
   }
 
   static async fromHeads(
@@ -246,7 +237,7 @@ module.exports = class FastForward {
       } catch (err) {
         safetyCatch(err)
         this.head = result.head
-        return { head: result.head, tip: null, migrate: result.migrate }
+        return { head: result.head, migrate: result.migrate }
       }
 
       // fall through to the plain fast-forward below
@@ -278,7 +269,6 @@ module.exports = class FastForward {
 
     return {
       head: this.head,
-      tip: this.tip,
       migrate: null
     }
   }
@@ -331,7 +321,6 @@ module.exports = class FastForward {
 
       return {
         head,
-        tip: null,
         migrate: views,
         autobee: autobee === null ? null : batchToHead(autobee.op.views.system)
       }

package/lib/migrations.js

@@ -37,23 +37,36 @@ async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
   const bootRecord = await local.getUserData('autobase/boot')
   if (!bootRecord) return null
 
-  const { key } = encoding.decodeAutobaseBootRecord(bootRecord)
+  const { key, systemLength } = encoding.decodeAutobaseBootRecord(bootRecord)
   const core = store.get({ key: key, encryption: null })
   await core.ready()
 
-  const system = { key, length: core.length }
+  const length = await findLocalSystemLength(core, systemLength)
+  assert(length > 0, 'Expected system block to exist locally')
+
+  const system = { key, length }
 
   // setup encryption
   const encryptionKey = await local.getUserData('autobase/encryption')
   await AutobeeEncryption.setSystemEncryption(bootstrap.key, encryptionKey, core)
 
-  const catchup = await getCatchupHeads(core)
+  const session = core.session({ name: 'batch' })
+  await session.ready()
+
+  let catchup
+
+  try {
+    catchup = await getCatchupHeads(session, length)
+  } finally {
+    await session.close()
+  }
+
+  const info = await readLegacySystemInfo(core, length, { wait: false })
+
   const nodes = await Promise.all(
     catchup.map((n) => getWriterBatch(store, n, bootstrap.key, encryptionKey))
   )
 
-  const info = await readLegacySystemInfo(core, core.length)
-
   const indexerManifests = await Promise.all(
     info.indexers.map((idx) => storeCoreManifest(store, idx.key))
   )
@@ -89,8 +102,11 @@ async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
 
 // decode the legacy system info recorded in the system core's block at length
 async function readLegacySystemInfo(core, length, opts = null) {
+  const block = await core.get(length - 1, opts)
+  assert(block !== null, 'Expected system block to exist locally')
+
   // Decode hyperbee block using hyperbee2 compat
-  const node = decodeBlock(await core.get(length - 1, opts))
+  const node = decodeBlock(block)
   assert(node.keys.length > 0, 'bad system block')
 
   // Decode system info from hyperbee block using autobee compat
@@ -259,6 +275,15 @@ function deriveNamespace(name, bootstrap, entropy, encryptionKey) {
   ])
 }
 
+// the record's length can run past the signed core into the local batch
+// session, and the signed core can carry verified-but-undownloaded blocks -
+// scan back for the newest block the signed core actually has locally
+async function findLocalSystemLength(core, systemLength) {
+  let length = Math.min(core.length, systemLength)
+  while (length > 0 && !(await core.has(length - 1))) length--
+  return length
+}
+
 function coreLength(core, timeout) {
   if (core.length) return core.length
 
@@ -355,18 +380,20 @@ async function getPersistedView(store, view) {
   const core = store.get(view.key)
   await core.ready()
 
-  const length = core.length
+  // core.length can run past what was downloaded - the system's recorded
+  // length is the last one we know is local
+  const length = Math.min(core.length, view.length)
   await core.close()
 
   return { key: view.key, length }
 }
 
-async function getCatchupHeads(core) {
-  const session = core.session({ name: 'batch' })
-  await session.ready()
-
+async function getCatchupHeads(session, from) {
+  const seen = new Map()
   const nodes = []
-  for (let i = core.length; i < session.length; i++) {
+
+  // block 0 is the bee header, never a node
+  for (let i = Math.max(from, 1); i < session.length; i++) {
     const node = await session.get(i, { wait: false })
     if (!node) throw new Error('Expect nodes to exist locally')
 
@@ -375,11 +402,16 @@ async function getCatchupHeads(core) {
 
     const { key, length } = encoding.decodeSystemWriter(entry.key, entry.value)
     if (!length) continue
+
+    const id = b4a.toString(key, 'hex')
+    const current = seen.get(id) || 0
+
+    if (current >= length) continue
+
+    seen.set(id, length)
     nodes.push({ key, length })
   }
 
-  await session.close()
-
   return nodes
 }
 

package/lib/topo.js

@@ -10,7 +10,6 @@ const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 const DEFAULT_OP_TIMEOUT = 5000
 
 exports.sort = sort
-exports.rollback = rollback
 exports.replay = replay
 exports.getMostRecentHead = getMostRecentHead
 
@@ -49,46 +48,6 @@ async function getMostRecentHead(auto, system) {
   return null
 }
 
-async function rollback(auto, system, verified) {
-  if (!(await isApplied(system, verified.op))) return null
-
-  const tip = []
-  const ch = system.createChangesStream()
-
-  let found = false
-
-  for await (const data of ch) {
-    const { ref, info } = getOplog(data)
-
-    if (sameNode(ref.oplog, verified.op)) {
-      // stale views.flushes - the verified head describes a prefix this system
-      // does not share, so nothing above it is ours to reapply
-      if (info.flushes === verified.flushes) found = true
-      break
-    }
-
-    // walking back, a writer's lengths only decrease - once this writer is
-    // behind the verified head the marker cannot be further down
-    if (b4a.equals(ref.oplog.key, verified.op.key) && ref.oplog.length < verified.op.length) {
-      break
-    }
-
-    tip.push(ref)
-  }
-
-  if (!found) return null
-
-  const inflated = await inflate(auto, tip.reverse())
-
-  // @todo: verify sort
-
-  return {
-    undo: null,
-    view: null,
-    tip: inflated.map(getBatch)
-  }
-}
-
 // nodes we cannot resolve are returned as null rather than dropped, so callers
 // can tell an incomplete replay from a short one
 async function replay(auto) {
@@ -109,14 +68,6 @@ async function replay(auto) {
   return inflated.flatMap((b) => (b === null ? [null] : b.batch))
 }
 
-async function isApplied(system, node) {
-  const entry = await system.get(encoding.encodeSystemWriterKey(node.key))
-  if (entry === null) return false
-
-  const info = encoding.decodeSystemWriter(entry.key, entry.value)
-  return info.length >= node.length
-}
-
 async function sort(auto, batch) {
   const node = batch[0]
   const ch = auto.system.bee.createChangesStream()
@@ -125,7 +76,7 @@ async function sort(auto, batch) {
   for await (const data of ch) {
     if (isLegacyNode(data)) break
 
-    const { ref, info } = getOplog(data)
+    const { ref } = getOplog(data)
     const link = {
       key: ref.oplog.key,
       length: ref.oplog.length,
@@ -448,11 +399,3 @@ function isLegacyNode(data) {
   }
   return false
 }
-
-function sameNode(a, b) {
-  return b4a.equals(a.key, b.key) && a.length === b.length
-}
-
-function getBatch(b) {
-  return b.batch
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.27",
+  "version": "2.0.0-rc.29",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",