autobee 2.2.2 → 2.3.0

package/README.md

@@ -247,17 +247,16 @@ Called at flush time to stamp your own oplog (with your view as `target` and a `
 
 ```js
 {
-  head: { key, length },       // oplog head to boot from
-  legacy: { key, length },     // pre-2.0 pointer, see below
-  bootCondition (target, reference) {} // optional gate on the view we would land on
+  head: { key, length },  // oplog head to boot from
+  legacy: { key, length } // pre-2.0 pointer, see below
 }
 ```
 
 Pass one of `head` or `legacy`, not both.
 
-Without `bootCondition` this is a single attempt that gives up if the head cannot be read; with one it parks until the condition is satisfied, which today means indefinitely if it never is.
+`head` must be a v3 or newer oplog node. `length` is a floor: the key is searched for its latest oplog head at or past it, and the boot lands ungated on the views that head stamps. It is a single attempt: use `moveTo` if you need to retry.
 
-`legacy` is for records written before boot heads were oplog heads: the key is a _system_ head and a `0` length is resolved from the core. It boots ungated - `bootCondition` does not apply - and will be removed, so don't reach for it. A bare `{ key, length }` in place of the whole struct means the same thing, older still.
+`legacy` is the pre-2.0 pointer: a _system_ head whose `0` length resolves from the core. It will be removed, so don't reach for it. A bare `{ key, length }` in place of the whole struct means the same.
 
 #### `fastForward.conservative`
 
@@ -265,9 +264,11 @@ Defaults to `true`: only fast-forward onto a head a connected peer can serve who
 
 The check covers the oplog head only, not the system and view cores the fast-forward then reads.
 
-#### `await db.moveTo(head)`
+#### `await db.moveTo(head, [options])`
 
-Fast-forward onto an oplog head, ignoring the usual distance and conservative checks. Resolves `{ to, from }`.
+Fast-forward onto an oplog head, ignoring the usual distance and conservative checks. Resolves `{ to, from }`, or `null` if the head could not be booted.
+
+Pass `{ timeout }` to bound the reads, so a head nobody can serve fails instead of hanging.
 
 ### Encryption
 

package/index.js

@@ -14,7 +14,6 @@ const asserts = require('./lib/asserts.js')
 const boot = require('./lib/boot.js')
 const { resolveWeight, currentWeight } = require('./lib/witness.js')
 const encoding = require('./lib/encoding.js')
-const { LEGACY_OPLOG_VERSION } = require('./lib/constants.js')
 const FastForward = require('./lib/fast-forward.js')
 const System = require('./lib/system.js')
 const ApplyCalls = require('./lib/apply-calls.js')
@@ -72,7 +71,7 @@ module.exports = class Autobee extends ReadyResource {
     this.stats = { undos: 0, fastForwards: 0, drains: 0, applies: 0, appends: 0 }
 
     const systemStore = this.store.session()
-    this.system = new System(systemStore, this.name, {
+    this.system = new System(this, systemStore, {
       core: systemStore.get({ preload: this._getCorePreload('system') }),
       encrypted: this.encrypted,
       getEncryptionProvider: this.getSystemEncryption
@@ -119,7 +118,6 @@ module.exports = class Autobee extends ReadyResource {
     this._appending = []
     this._draining = null
     this._updating = null
-    this._bootWait = null
 
     this.legacyViews = handlers.legacyViews || []
 
@@ -145,7 +143,6 @@ module.exports = class Autobee extends ReadyResource {
     this.interrupted = null
     this._interrupting = false
     this._onErrorBound = this._onError.bind(this)
-    this._bumpSoonBound = this.bumpSoon.bind(this)
     this._onGroupUpdateBound = this._onGroupUpdate.bind(this)
 
     this.wakeupCapability = null
@@ -153,7 +150,7 @@ module.exports = class Autobee extends ReadyResource {
     this.previousDrain = 0
 
     this._catchupMigratedNodes = null
-    this._migratedHead = null
+    this._migrating = false
     this._prebooting = null
 
     this.ready().catch(noop)
@@ -393,7 +390,6 @@ module.exports = class Autobee extends ReadyResource {
     // rugpull the rest
     await this.store.close()
 
-    if (this._bootWait !== null) this._bootWait.resolve()
     if (this._updating) await this._updating
     if (this._draining) await this._draining
 
@@ -437,10 +433,6 @@ module.exports = class Autobee extends ReadyResource {
     return new WriterEncryption(this)
   }
 
-  getMostRecentHead() {
-    return topo.getMostRecentHead(this, this.system.bee.snapshot())
-  }
-
   async _preBoot() {
     if (this._handlers.wait) await this._handlers.wait()
 
@@ -458,7 +450,7 @@ module.exports = class Autobee extends ReadyResource {
       this.bootFrom = getBootOption(await this.bootFrom)
     }
 
-    return boot(this.store, this.key, this.legacyViews, {
+    return boot(this.store, this.key, {
       encryptionKey: this.encryptionKey,
       keyPair: this.keyPair
     })
@@ -505,37 +497,15 @@ module.exports = class Autobee extends ReadyResource {
     const system = result.system || EMPTY_HEAD
 
     await this.system.boot(system)
+    if (this.system.migration) await this._runMigration()
 
-    const migrated = await this.local.getUserData('autobee/migrated-head')
-    if (migrated) this._migratedHead = encoding.decodeMigratedHead(migrated)
-
-    let view = this.system.view
-    if (!view) view = (this._migratedHead && this._migratedHead.view) || EMPTY_HEAD
-
-    // @todo migration
     if (result.migration) {
-      if (this._handlers.migrate) {
-        view =
-          (await this._handlers.migrate(result.migration.views, result.migration.system)) ||
-          EMPTY_HEAD
-        this._catchupMigratedNodes = result.migration.catchup
-
-        this._migratedHead = {
-          system: result.migration.system,
-          view: view.length ? view : (this._migratedHead && this._migratedHead.view) || null
-        }
-
-        await this._storeMigratedHead()
-      }
+      this._migrating = true
+      this._catchupMigratedNodes = result.migration.catchup
 
       // ff boot invalidated by migration
       this.bootFrom = null
 
-      // clear legacy data
-      await this.bootstrap.setUserData('autobase/local', null)
-      await this.local.setUserData('autobase/boot', null)
-      await this.local.setUserData('autobase/encryption', null)
-
       for (const batch of result.migration.catchup) {
         const { key, length } = batch[batch.length - 1]
         this.writers.wakeup(key, length)
@@ -547,6 +517,8 @@ module.exports = class Autobee extends ReadyResource {
     await this._workingBee.core.ready()
     await this.bee.core.ready()
 
+    const view = this.system.view
+
     this._workingBee.move(view)
     this.bee.move(view)
 
@@ -592,10 +564,6 @@ module.exports = class Autobee extends ReadyResource {
   async _bump(force) {
     if (!force && !this._bootGuard.opened) await this._bootGuard.ready()
 
-    // resolve before the bootAll gate: a parked boot-from retries on any bump,
-    // and its drain is the one _bootOnline itself is waiting on
-    if (this._bootWait !== null) this._bootWait.resolve()
-
     if (!force && !this._bootOnlineGuard.opened) await this._bootOnlineGuard.ready()
 
     this.bumping++
@@ -668,12 +636,16 @@ module.exports = class Autobee extends ReadyResource {
     this.stats.drains++
 
     if (this.bootFrom) {
-      const { head = null, legacy = null, bootCondition = null, wait = false } = this.bootFrom
+      const { head = null, legacy = null } = this.bootFrom
 
       this.bootFrom = null
 
-      if (legacy) await this._bootFromSystem(legacy)
-      else if (head) await this._bootFromHead(head, bootCondition, wait)
+      if (legacy) {
+        await this._bootFromSystem(legacy)
+      } else if (head) {
+        this._wakeup.hint({ key: head.key, length: head.length || 0 })
+        await this._bootFromHead(head)
+      }
     }
 
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
@@ -695,6 +667,8 @@ module.exports = class Autobee extends ReadyResource {
 
           if (this.fastForwardTo !== null) {
             await this._applyFastForward()
+            if (changes) changes.track()
+            this._needsUpdate = false
             break // revaluate conditions...
           }
 
@@ -718,7 +692,7 @@ module.exports = class Autobee extends ReadyResource {
     this._updating = updating.promise
 
     try {
-      if (this._needsUpdate) await this._update(changes)
+      if (this._needsUpdate) await this._update(changes, false)
       await this._storeBoot()
     } finally {
       this._updating = null
@@ -900,13 +874,13 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _update(changes) {
+  async _update(changes, fastForward) {
     this._needsUpdate = false
     this.bee.update(this._workingBee.root)
 
     if (!changes) return
 
-    changes.finalise()
+    changes.finalise(fastForward)
     await this._handlers.update(this.view, changes)
   }
 
@@ -1205,9 +1179,6 @@ module.exports = class Autobee extends ReadyResource {
       this.stats.undos++
       this.trusted.clear()
       t.view = await this.system.undo(t.undo)
-      if (!t.view.length && this._migratedHead && this._migratedHead.view) {
-        t.view = this._migratedHead.view
-      }
     }
 
     return t
@@ -1288,8 +1259,9 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  _storeBoot() {
+  async _storeBoot() {
     const proms = []
+
     proms.push(
       this.local.setUserData(
         'autobee/previous-drain',
@@ -1302,12 +1274,15 @@ module.exports = class Autobee extends ReadyResource {
       proms.push(this.local.setUserData('autobee/head', encoding.encodeBootRecord(boot)))
     }
 
-    return Promise.all(proms)
-  }
+    await Promise.all(proms)
 
-  _storeMigratedHead() {
-    const value = this._migratedHead ? encoding.encodeMigratedHead(this._migratedHead) : null
-    return this.local.setUserData('autobee/migrated-head', value)
+    if (this._migrating) {
+      // clear legacy data
+      await this.bootstrap.setUserData('autobase/local', null)
+      await this.local.setUserData('autobase/boot', null)
+      await this.local.setUserData('autobase/encryption', null)
+      this._migrating = false
+    }
   }
 
   static decodeValue(buf, opts) {
@@ -1404,11 +1379,11 @@ module.exports = class Autobee extends ReadyResource {
     this._localViewLength = 0
   }
 
-  async moveTo(head) {
+  async moveTo(head, { timeout = 0 } = {}) {
     if (!this._bootGuard.opened) await this._bootGuard.ready()
     if (this.closing) throw new Error('Autobee closed')
 
-    const ff = await FastForward.fromHead(this, head, null, { force: true })
+    const ff = await FastForward.fromHead(this, head, null, { force: true, timeout })
     if (ff === null) return null
 
     if (!(await this._runFastForward(ff))) return null
@@ -1427,108 +1402,20 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  // the boot head seeds the view every trust decision here is made against,
-  // since our own view is still empty
-  async _bootFromHead(head, bootCondition, wait = false) {
-    // just a head: one attempt, we do not wait around if it cannot be read
-    if (bootCondition === null && !wait) {
-      try {
-        const ff = await FastForward.fromHead(this, head, null, {
-          force: true,
-          timeout: FastForward.DEFAULT_TIMEOUT
-        })
-
-        return ff !== null && (await this._runFastForward(ff))
-      } catch (err) {
-        safetyCatch(err)
-        return false
-      }
-    }
-
-    // park until the head can be read and, if a condition is set, something
-    // acceptable turns up
-    let opened = null
+  async _bootFromHead(head) {
+    const timeout = FastForward.DEFAULT_TIMEOUT
 
     try {
-      while (!this._interrupting) {
-        this._bootWait = rrp()
-
-        try {
-          if (bootCondition === null) {
-            const ff = await FastForward.fromHead(this, head, null, {
-              force: true,
-              timeout: FastForward.DEFAULT_TIMEOUT
-            })
-
-            if (ff !== null && (await this._runFastForward(ff))) return true
-          } else {
-            if (opened === null) opened = await this._bootReference(head)
-
-            if (opened !== null) {
-              if (await this._bootAttempt(head, bootCondition, opened.view)) {
-                return true
-              }
-
-              // the reference may be stale - resolve it afresh next attempt
-              await opened.close()
-              opened = null
-            } else if (await this._bootLegacyHead(head)) {
-              // boot legacy head through the migration pathway
-              return true
-            }
-          }
-        } catch (err) {
-          safetyCatch(err)
-        }
+      const oplog = await this._resolveOplogHint(head.key, head.length || 0, { timeout })
+      if (oplog === null) return false
 
-        if (this._interrupting) break
+      const ff = await FastForward.fromHead(this, oplog, null, { force: true, timeout })
 
-        await this._bootWait.promise
-      }
-    } finally {
-      this._bootWait = null
-      if (opened !== null) await opened.close()
+      return ff !== null && (await this._runFastForward(ff))
+    } catch (err) {
+      safetyCatch(err)
+      return false
     }
-
-    return false
-  }
-
-  async _bootAttempt(head, bootCondition, reference) {
-    // peek, so the hints are still there for the drain once we have booted
-    const result = await this._filterHints(this._wakeup.hints)
-
-    const candidates = await this._readCandidateHeads(result, FastForward.DEFAULT_TIMEOUT)
-
-    const ff = await FastForward.fromHeads(this, [head, ...candidates], {
-      force: true,
-      timeout: FastForward.DEFAULT_TIMEOUT,
-      condition: bootCondition,
-      reference
-    })
-
-    return ff !== null && (await this._runFastForward(ff))
-  }
-
-  async _bootLegacyHead(head) {
-    const oplog = await FastForward.flushHead(this, head, {
-      timeout: FastForward.DEFAULT_TIMEOUT
-    })
-    if (oplog === null || oplog.op.version > LEGACY_OPLOG_VERSION) return false
-
-    const ff = await FastForward.fromHead(this, head, null, {
-      force: true,
-      timeout: FastForward.DEFAULT_TIMEOUT
-    })
-
-    return ff !== null && (await this._runFastForward(ff))
-  }
-
-  async _bootReference(head) {
-    const oplog = await FastForward.flushHead(this, head, {
-      timeout: FastForward.DEFAULT_TIMEOUT
-    })
-
-    return oplog === null ? null : this.openViewAt(oplog)
   }
 
   async _runFastForward(ff) {
@@ -1554,34 +1441,32 @@ module.exports = class Autobee extends ReadyResource {
     return true
   }
 
+  async _runMigration() {
+    if (!this._handlers.migrate) {
+      throw new Error('Missing migration handler')
+    }
+
+    const migration = this.system.migration
+    this.system.migration = null
+
+    await this._handlers.migrate(migration.views, migration.head)
+  }
+
   async _applyFastForward() {
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
-    const { head, migrate } = this.fastForwardTo
+    const { head } = this.fastForwardTo
 
     const from = this.system.bee.head()
     const to = head
 
     this.system.bee.move(head)
     await this.system.reset()
+    if (this.system.migration) await this._runMigration()
 
-    // migrate is set when fast-forwarding from a legacy head
-    if (migrate) {
-      const view = (await this._handlers.migrate(migrate, head)) || EMPTY_HEAD
-
-      this._migratedHead = {
-        system: head,
-        view: view.length ? view : (this._migratedHead && this._migratedHead.view) || null
-      }
-
-      await this._storeMigratedHead()
-      this.bee.move(view)
-      this._workingBee.move(view)
-    } else {
-      this.bee.move(this.system.view)
-      this._workingBee.move(this.system.view)
-    }
+    this.bee.move(this.system.view)
+    this._workingBee.move(this.system.view)
 
     this._approvalCheck = true
     this.fastForwardTo = null
@@ -1593,7 +1478,7 @@ module.exports = class Autobee extends ReadyResource {
     // we moved, so ask our peers to tell us their heads again
     this._requestWakeup()
 
-    await this._update(changes)
+    await this._update(changes, true)
     await this._storeBoot()
 
     this.stats.fastForwards++
@@ -1614,7 +1499,7 @@ function getBootOption(boot) {
   if (!boot) return null
 
   // oldest style, supported for now but will go away: a bare key is legacy
-  if (boot.key) return { legacy: boot, head: null, bootCondition: null }
+  if (boot.key) return { legacy: boot, head: null }
 
   asserts.assert(!(boot.head && boot.legacy), 'Boot from either a head or a legacy pointer')
 

package/lib/boot.js

@@ -5,7 +5,6 @@ const { checkAutobaseMigration } = require('./migrations.js')
 module.exports = async function boot(
   corestore,
   key,
-  legacyViews,
   { encrypt, encryptionKey, keyPair, exclusive = true } = {}
 ) {
   const result = {
@@ -95,12 +94,7 @@ module.exports = async function boot(
     await result.bootstrap.setUserData('autobee/local', result.local.key)
     await result.local.setUserData('referrer', result.key)
 
-    result.migration = await migrateWithFallback(
-      corestore,
-      result.local,
-      result.bootstrap,
-      legacyViews
-    )
+    result.migration = await migrateWithFallback(corestore, result.local, result.bootstrap)
 
     if (result.migration) {
       await result.local.setUserData(
@@ -142,9 +136,9 @@ module.exports = async function boot(
 // the bootstrap pointer names the rotated-to writer, whose legacy boot record
 // can be unusable. the namespace local still carries the pre-rotation record,
 // which points at the last system core that was actually signed
-async function migrateWithFallback(corestore, local, bootstrap, legacyViews) {
+async function migrateWithFallback(corestore, local, bootstrap) {
   try {
-    return await checkAutobaseMigration(corestore, local, bootstrap, legacyViews)
+    return await checkAutobaseMigration(corestore, local, bootstrap)
   } catch (err) {
     const prev = corestore.get({ name: 'local', active: false })
     await prev.ready()
@@ -152,7 +146,7 @@ async function migrateWithFallback(corestore, local, bootstrap, legacyViews) {
     if (b4a.equals(prev.key, local.key)) throw err
     if (!(await prev.getUserData('autobase/boot'))) throw err
 
-    return checkAutobaseMigration(corestore, prev, bootstrap, legacyViews)
+    return checkAutobaseMigration(corestore, prev, bootstrap)
   }
 }
 

package/lib/encoding.js

@@ -12,19 +12,16 @@ const SystemInfo = getEncoding('@autobee/system-info')
 const SystemWriter = getEncoding('@autobee/system-writer')
 const ManifestData = getEncoding('@autobee/manifest-data')
 const AutobaseBootRecord = getEncoding('@autobase-compat/boot-record')
-const MigratedHead = getEncoding('@autobee/migrated-head')
 
 exports.OPLOG_VERSION = OPLOG_VERSION
 
 exports.Oplog = Oplog
+
 exports.ManifestData = ManifestData
 
 exports.encodeBootRecord = encodeBootRecord
 exports.decodeBootRecord = decodeBootRecord
 
-exports.encodeMigratedHead = encodeMigratedHead
-exports.decodeMigratedHead = decodeMigratedHead
-
 exports.encodePreviousDrain = encodePreviousDrain
 exports.decodePreviousDrain = decodePreviousDrain
 
@@ -50,14 +47,6 @@ function decodeBootRecord(value) {
   return c.decode(Link, value)
 }
 
-function encodeMigratedHead(m) {
-  return c.encode(MigratedHead, m)
-}
-
-function decodeMigratedHead(value) {
-  return c.decode(MigratedHead, value)
-}
-
 function encodePreviousDrain(d) {
   return c.encode(c.uint, d)
 }

package/lib/fast-forward.js

@@ -1,10 +1,8 @@
 const safetyCatch = require('safety-catch')
-const b4a = require('b4a')
 const Hypercore = require('hypercore')
 
 const System = require('./system.js')
 const migrations = require('./migrations.js')
-const { LEGACY_AUTOBASE_VERSION } = require('./constants.js')
 
 const DEFAULT_OP_TIMEOUT = 5_000
 const MIN_FF_GAP = 32
@@ -17,7 +15,7 @@ module.exports = class FastForward {
 
     // 'reboot' is a storage namespace: renaming it would derive a different
     // scratch core on disk, so it stays as is
-    this.system = new System(auto.store.namespace('reboot'), null, {
+    this.system = new System(auto, auto.store.namespace('reboot'), {
       getEncryptionProvider: this.auto.getSystemEncryption,
       encrypted: this.encrypted,
       activeRequests: this.activeRequests
@@ -34,12 +32,7 @@ module.exports = class FastForward {
 
   static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
 
-  static async fromHead(
-    auto,
-    head,
-    trusted,
-    { force = false, timeout = 0, condition = null, reference = null } = {}
-  ) {
+  static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
     const conservative = !force && auto._conservativeFF
     const timeoutOpts = timeout ? { timeout } : null
 
@@ -59,19 +52,11 @@ module.exports = class FastForward {
       return null
     }
 
-    if (condition !== null && !(await FastForward.accepts(auto, verified, condition, reference))) {
-      return null
-    }
-
     return new FastForward(auto, batchToHead(verified.op.views.system))
   }
 
-  static async fromHeads(
-    auto,
-    heads,
-    { force = false, timeout = 0, condition = null, reference = null } = {}
-  ) {
-    if (reference === null) reference = auto._workingView.view
+  static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
+    const reference = auto._workingView.view
 
     const promises = []
 
@@ -112,12 +97,7 @@ module.exports = class FastForward {
     }
 
     if (bestTrusted !== null) {
-      return FastForward.fromHead(auto, bestTrusted, null, {
-        force,
-        timeout,
-        condition,
-        reference
-      })
+      return FastForward.fromHead(auto, bestTrusted, null, { force, timeout })
     }
 
     candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
@@ -127,12 +107,7 @@ module.exports = class FastForward {
       if (trusted === null) continue
       if (auto.fastForwarding || auto.fastForwardTo) return null
 
-      const ff = await FastForward.fromHead(auto, res, trusted, {
-        force,
-        timeout,
-        condition,
-        reference
-      })
+      const ff = await FastForward.fromHead(auto, res, trusted, { force, timeout })
       if (ff !== null) return ff
     }
 
@@ -151,17 +126,6 @@ module.exports = class FastForward {
     }
   }
 
-  static async accepts(auto, oplog, condition, reference) {
-    const opened = auto.openViewAt(oplog)
-    if (opened === null) return false
-
-    try {
-      return !!(await condition(opened.view, reference))
-    } finally {
-      await opened.close()
-    }
-  }
-
   static async mostRecentTrusted(auto, head, reference) {
     const opened = auto.openViewAt(head)
     if (opened === null) return null
@@ -198,38 +162,6 @@ module.exports = class FastForward {
 
     await this.system.boot(this.head, { timeout: this.timeout })
 
-    // a legacy system needs migrating rather than a plain fast-forward
-    if (this.system.version <= LEGACY_AUTOBASE_VERSION) {
-      const migrated = this.auto._migratedHead && this.auto._migratedHead.system
-      if (
-        migrated &&
-        b4a.equals(migrated.key, this.head.key) &&
-        migrated.length >= this.head.length
-      ) {
-        return null
-      }
-
-      const result = await this._migrate()
-      if (result === null || result.autobee === null) return result
-
-      // an indexer of the newest legacy generation already moved to autobee -
-      // fast-forward straight onto its announced head instead of migrating,
-      // falling back to the legacy migration if the head cannot be booted
-      try {
-        this.head = result.autobee
-        await this.system.boot(this.head, { timeout: this.timeout })
-        if (this.system.version <= LEGACY_AUTOBASE_VERSION) {
-          throw new Error('Expected an autobee system')
-        }
-      } catch (err) {
-        safetyCatch(err)
-        this.head = result.head
-        return { head: result.head, migrate: result.migrate }
-      }
-
-      // fall through to the plain fast-forward below
-    }
-
     const promises = []
 
     // ensure local key is locally available always
@@ -252,70 +184,12 @@ module.exports = class FastForward {
     await Promise.all(promises)
     if (this.destroyed) return null
 
-    if (this.destroyed) return null
-
     return {
       head: this.head,
       migrate: null
     }
   }
 
-  // Upgrade a legacy (pre-AUTOBEE_VERSION) system in place. The system already
-  // booted above (coerced to the current struct), but the original v1/v2 struct
-  // is needed to recover the indexers/views/entropy, so decode the raw block.
-  //
-  // A head may be arbitrarily old: an indexer rotation freezes the system core
-  // and moves the indexers to a successor. The frozen core cannot reveal its
-  // successor (the promotion only lands in its unsigned tail), but the
-  // indexers' newest oplog digests stamp the newest system they know - chase
-  // those so the migration lands on the newest generation instead of the head
-  // we were given.
-  async _migrate() {
-    let head = { key: this.head.key, length: this.head.length }
-    const seen = new Set()
-
-    for (let hops = 0; hops < migrations.MAX_MIGRATE_HOPS; hops++) {
-      seen.add(b4a.toString(head.key, 'hex'))
-
-      const core = this.auto.store.get({ key: head.key, encryption: null })
-      this.cores.push(core)
-      await core.ready()
-
-      // setup encryption
-      await core.setEncryption(this.auto.getSystemEncryption())
-
-      // successors are resolved by key only - wait for the replicated length
-      if (!head.length) head.length = await migrations.coreLength(core, this.timeout)
-      if (!head.length || this.destroyed) return null
-
-      const info = await migrations.readLegacySystemInfo(core, head.length, {
-        timeout: this.timeout,
-        activeRequests: this.activeRequests
-      })
-
-      const { legacy, autobee } = await migrations.findNewestSystem(this, info)
-      if (this.destroyed) return null
-
-      if (legacy !== null && !seen.has(b4a.toString(legacy, 'hex'))) {
-        head = { key: legacy, length: 0 }
-        continue
-      }
-
-      this.head = head
-
-      const views = await migrations.resolveLegacyViews(this, info)
-      if (views === null || this.destroyed) return null
-
-      return {
-        head,
-        migrate: views,
-        autobee: autobee === null ? null : batchToHead(autobee.op.views.system)
-      }
-    }
-
-    return null
-  }
-
   async _resolveLength() {
     const core = this.auto.store.get({ key: this.head.key })
     this.cores.push(core)

package/lib/migrations.js

@@ -6,7 +6,6 @@ const { decodeBlock } = require('hyperbee2/lib/encoding.js')
 
 const encoding = require('./encoding.js')
 const { assert, bail } = require('./asserts.js')
-const { LEGACY_OPLOG_VERSION } = require('./constants.js')
 
 const { getEncoding } = require('../encoding/spec/autobee')
 
@@ -18,23 +17,52 @@ const INDEX_VERSION = 1
 const INFO = b4a.from([0x00, 0x00, 0x69, 0x6e, 0x66, 0x6f])
 const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
 
-// indexer rotations chain legacy systems - how many generations we chase
-const MAX_MIGRATE_HOPS = 64
-
 module.exports = {
-  MAX_MIGRATE_HOPS,
+  getCoreManifest,
+  getViewMap,
   checkAutobaseMigration,
-  readLegacySystemInfo,
-  findNewestSystem,
-  resolveLegacyViews,
   inflateLegacyOplog,
-  decodeLegacySystemInfo,
-  deriveNamespace,
   coreLength
 }
 
+async function getCoreManifest(store, key, { timeout, wait = true } = {}) {
+  const core = store.get(key)
+  await core.ready()
+  try {
+    if (!core.manifest && core.length === 0) await core.get(0, { timeout, wait })
+    return core.manifest
+  } finally {
+    await core.close()
+  }
+}
+
+// local-only probes: the manifests are expected in storage already (prefetched
+// by the reset path), so a violation skips instead of hanging on the network
+async function getViewMap(store, bootstrap, encryptionKey, info, legacyViews) {
+  const getIndexerManifest = ({ key }) => getCoreManifest(store, key, { wait: false })
+  const indexerManifests = await Promise.all(info.indexers.map(getIndexerManifest))
+  const entropy = legacyEntropy(info, indexerManifests)
+  const mappedViews = new Map()
+
+  for (const name of legacyViews) {
+    const v = await findViewByName(
+      store,
+      bootstrap.key,
+      encryptionKey,
+      indexerManifests,
+      info.views,
+      entropy,
+      name
+    )
+
+    mappedViews.set(name, v)
+  }
+
+  return mappedViews
+}
+
 // boot-time migration of local legacy autobase storage
-async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
+async function checkAutobaseMigration(store, local, bootstrap) {
   const bootRecord = await local.getUserData('autobase/boot')
   if (!bootRecord) return null
 
@@ -60,41 +88,19 @@ async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
     await session.close()
   }
 
-  const info = await readLegacySystemInfo(core, length, { wait: false })
+  // nothing reads the info here anymore, but a legacy system that will not
+  // decode has to throw for migrateWithFallback pathway
+  await readLegacySystemInfo(core, length, { wait: false })
 
   const nodes = await Promise.all(
     catchup.map((n) => getWriterBatch(store, n, bootstrap.key, encryptionKey))
   )
 
-  const indexerManifests = await Promise.all(
-    info.indexers.map((idx) => storeCoreManifest(store, idx.key))
-  )
-  const views = await Promise.all(info.views.map((v) => getPersistedView(store, v)))
-  const entropy = legacyEntropy(info, indexerManifests)
-
-  const getManifest = (key) => storeCoreManifest(store, key)
-
-  const mappedViews = new Map()
-  for (const name of legacyViews) {
-    const v = await findViewByName(
-      getManifest,
-      bootstrap.key,
-      encryptionKey,
-      indexerManifests,
-      views,
-      entropy,
-      name
-    )
-
-    mappedViews.set(name, v)
-  }
-
   await core.close()
 
   return {
     encryptionKey,
     system,
-    views: mappedViews,
     catchup: nodes
   }
 }
@@ -112,95 +118,6 @@ async function readLegacySystemInfo(core, length, opts = null) {
   return decodeLegacySystemInfo(node.keys[0].value)
 }
 
-// read each indexer's newest legacy oplog stamp - its digest/checkpoint mark
-// the newest legacy system that indexer knows. take the majority vote. an
-// indexer that already moved to autobee announces its live autobee state
-// instead - keep the newest of those as a direct fast-forward candidate and
-// bisect down to its buried legacy stamps so its vote still counts
-async function findNewestSystem(ff, info) {
-  const tally = new Map()
-
-  let autobee = null
-
-  for (const idx of info.indexers) {
-    const core = ff.auto.openCore(idx.key)
-    ff.cores.push(core)
-    await core.ready()
-
-    const length = await coreLength(core, ff.timeout)
-    if (!length || ff.destroyed) continue
-
-    let oplog = await flushHead(ff.auto, { key: idx.key, length }, { timeout: ff.timeout })
-    if (oplog === null || !oplog.op.views) continue
-
-    if (oplog.op.version > LEGACY_OPLOG_VERSION) {
-      if (autobee === null || oplog.op.views.flushes > autobee.op.views.flushes) {
-        autobee = oplog
-      }
-
-      const boundary = await findLegacyBoundary(ff, core, idx.length, oplog.length - 1)
-      if (!boundary || ff.destroyed) continue
-
-      oplog = await flushHead(ff.auto, { key: idx.key, length: boundary }, { timeout: ff.timeout })
-      if (oplog === null || !oplog.op.views) continue
-      if (oplog.op.version > LEGACY_OPLOG_VERSION) continue
-    }
-
-    const sys = oplog.op.views.system
-    if (!sys || !sys.key) continue
-
-    const hex = b4a.toString(sys.key, 'hex')
-    const t = tally.get(hex) || { key: sys.key, votes: 0, length: sys.length }
-
-    t.votes++
-    if (sys.length > t.length) t.length = sys.length
-    tally.set(hex, t)
-  }
-
-  const majority = (info.indexers.length >> 1) + 1
-
-  let best = null
-  for (const t of tally.values()) {
-    if (t.votes >= majority && (best === null || t.length > best.length)) best = t
-  }
-
-  return { legacy: best === null ? null : best.key, autobee }
-}
-
-// map the legacy views recorded in the system info by name
-async function resolveLegacyViews(ff, info) {
-  const indexerManifests = await Promise.all(
-    info.indexers.map((idx) => ffCoreManifest(ff, idx.key, idx.length))
-  )
-  if (ff.destroyed || indexerManifests.some((m) => m === null)) return null
-
-  const entropy = legacyEntropy(info, indexerManifests)
-
-  const getManifest = (key, length) => ffCoreManifest(ff, key, length)
-
-  const views = new Map()
-
-  await Promise.all(
-    ff.auto.legacyViews.map(async (name) => {
-      const v = await findViewByName(
-        getManifest,
-        ff.auto.key,
-        ff.auto.encryptionKey,
-        indexerManifests,
-        info.views,
-        entropy,
-        name
-      )
-
-      if (v) views.set(name, v)
-    })
-  )
-
-  if (ff.destroyed) return null
-
-  return views
-}
-
 // legacy nodes always inflate, but only indexers carry the digest/checkpoint
 // stamps that synthesise system info - callers must check op.views
 async function inflateLegacyOplog(buf, core, seq, timeout) {
@@ -309,32 +226,12 @@ function coreLength(core, timeout) {
   })
 }
 
-// a writer's oplog is a legacy prefix followed by autobee-format nodes -
-// bisect for the boundary. lo is a length known to be legacy (the legacy
-// system's recorded length for this writer), hi a seq known to be autobee.
-// returns the legacy prefix length
-async function findLegacyBoundary(ff, core, lo, hi) {
-  while (lo < hi) {
-    const mid = (lo + hi) >>> 1
-
-    const block = await core.get(mid, {
-      timeout: ff.timeout,
-      activeRequests: ff.activeRequests
-    })
-
-    if (encoding.decodeRawOplog(block).version > LEGACY_OPLOG_VERSION) hi = mid
-    else lo = mid + 1
-  }
-
-  return lo
-}
-
 function legacyEntropy(info, indexerManifests) {
   return info.version > 1 && info.entropy ? info.entropy : indexerManifests[0].signers[0].namespace
 }
 
 async function findViewByName(
-  getManifest,
+  store,
   bootstrap,
   encryptionKey,
   indexerManifests,
@@ -347,7 +244,7 @@ async function findViewByName(
   const namespace = deriveNamespace(name, bootstrap, entropy, encryptionKey)
 
   for (const v of views) {
-    const manifest = await getManifest(v.key, v.length)
+    const manifest = await getCoreManifest(store, v.key, { wait: false })
     if (!manifest) continue
 
     if (manifest.signers.length === 0) continue
@@ -360,50 +257,6 @@ async function findViewByName(
   return null
 }
 
-// request-tracked manifest read over replication
-async function ffCoreManifest(ff, key, length) {
-  const core = ff.auto.store.get(key)
-  ff.cores.push(core)
-
-  await core.ready()
-
-  try {
-    if (!core.manifest) {
-      await core.get(length - 1, {
-        timeout: ff.timeout,
-        activeRequests: ff.activeRequests
-      })
-    }
-  } catch {
-    return null
-  }
-
-  return core.manifest
-}
-
-// local-storage manifest read
-async function storeCoreManifest(store, key) {
-  const core = store.get(key)
-  await core.ready()
-
-  const manifest = core.manifest
-  await core.close()
-
-  return manifest
-}
-
-async function getPersistedView(store, view) {
-  const core = store.get(view.key)
-  await core.ready()
-
-  // core.length can run past what was downloaded - the system's recorded
-  // length is the last one we know is local
-  const length = Math.min(core.length, view.length)
-  await core.close()
-
-  return { key: view.key, length }
-}
-
 function getOplog(info, batch) {
   if (batch.length === 1) return batch[0].keys[0]
 
@@ -489,9 +342,3 @@ async function getWriterBatch(store, head, key, encryptionKey, nodes = []) {
 
   return batch
 }
-
-// lazy to avoid the require cycle - fast-forward imports this module
-function flushHead(auto, head, opts) {
-  const FastForward = require('./fast-forward.js')
-  return FastForward.flushHead(auto, head, opts)
-}

package/lib/system.js

@@ -2,7 +2,9 @@ const Hyperbee = require('hyperbee2')
 const Promotions = require('./promotions.js')
 const b4a = require('b4a')
 
-const { AUTOBEE_VERSION } = require('./constants')
+const { getViewMap, getCoreManifest } = require('./migrations.js')
+
+const { AUTOBEE_VERSION, LEGACY_AUTOBASE_VERSION } = require('./constants')
 const encoding = require('./encoding.js')
 const topo = require('./topo.js')
 
@@ -33,12 +35,13 @@ class SystemSnapshot {
 }
 
 module.exports = class Systembee {
-  constructor(store, name, opts = {}) {
-    this.name = name // for debuggin
+  constructor(auto, store, opts = {}) {
+    this.auto = auto
     this.store = store
     this.bee = new Hyperbee(store, opts)
     this.view = null
     this.heads = []
+    this.migration = null
     this.version = 0
     this.timestamp = 0
     this.flushes = 0
@@ -48,6 +51,10 @@ module.exports = class Systembee {
     this.promotions = new Promotions(this.bee)
   }
 
+  get name() {
+    return this.auto.name
+  }
+
   async addWriter(
     key,
     { length = 0, weight = 1, isGenesis = false, coord = null, carrier = -1, anchor = false } = {}
@@ -127,9 +134,17 @@ module.exports = class Systembee {
 
     this.version = info ? info.version : 0
     this.view = info ? info.view : EMPTY_HEAD
+    this.migration = null
     this.heads = info ? info.heads : []
     this.timestamp = info ? info.timestamp : 0
     this.flushes = info ? info.flushes : 0
+
+    if (this.version <= LEGACY_AUTOBASE_VERSION) {
+      // ~~~ good enough estimation
+      this.flushes = Math.floor(this.bee.core.length / 2)
+      this.view = await this._migrate(info, { timeout })
+    }
+
     this.updates.clear()
     this.writers.clear()
     this.promotions.reset(info)
@@ -139,6 +154,29 @@ module.exports = class Systembee {
     }
   }
 
+  async _migrate(info, { timeout } = {}) {
+    if (!info) return EMPTY_HEAD
+    const a = this.auto
+
+    const toManifests = ({ key }) => getCoreManifest(a.store, key, { timeout })
+    const promises = [...info.indexers.map(toManifests), ...info.views.map(toManifests)]
+
+    await Promise.all(promises)
+
+    const views = await getViewMap(a.store, a.bootstrap, a.encryptionKey, info, a.legacyViews)
+
+    // the handler only runs once the head is locked in - see auto._runMigration
+    this.migration = { views, head: this.bee.head() }
+
+    // legacyViews is ordered by preference, so the first one that resolved wins
+    for (const name of a.legacyViews) {
+      const view = views.get(name)
+      if (view) return view
+    }
+
+    return EMPTY_HEAD
+  }
+
   bootRecord() {
     const system = this.bee.head()
     return system.length ? system : null
@@ -441,21 +479,3 @@ module.exports = class Systembee {
     return this.promotions.hint(key, opts)
   }
 }
-
-function getInfo(data) {
-  for (const { keys } of data.batch) {
-    for (const k of keys) {
-      const prefix = k.key[0]
-
-      if (prefix === 0) {
-        try {
-          return encoding.decodeSystemInfo(k.value)
-        } catch {
-          return null
-        }
-      }
-    }
-  }
-
-  return null
-}

package/lib/topo.js

@@ -7,15 +7,11 @@ const { LEGACY_OPLOG_VERSION } = require('./constants.js')
 const EMPTY_HEAD = { length: 0, key: null }
 const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 
-const DEFAULT_OP_TIMEOUT = 5000
-
 exports.sort = sort
 exports.replay = replay
-exports.getMostRecentHead = getMostRecentHead
 
 exports.isLinking = isLinking
 exports.isLinkingAll = isLinkingAll
-exports.isLegacyNode = isLegacyNode
 
 class Clock {
   constructor() {
@@ -35,19 +31,6 @@ class Clock {
   }
 }
 
-async function getMostRecentHead(auto, system) {
-  const ch = system.createChangesStream()
-  for await (const data of ch) {
-    const refs = getOplogCompat(data)
-    if (!refs.length) continue
-
-    const { oplog } = refs[0]
-    return { key: oplog.key, length: oplog.length }
-  }
-
-  return null
-}
-
 // nodes we cannot resolve are returned as null rather than dropped, so callers
 // can tell an incomplete replay from a short one
 async function replay(auto) {
@@ -181,7 +164,7 @@ function cmp(a, b) {
   return a.length < b.length ? -1 : 1
 }
 
-async function inflate(auto, reorder, { wait = true, timeout = DEFAULT_OP_TIMEOUT } = {}) {
+async function inflate(auto, reorder, { wait = true, timeout = 0 } = {}) {
   const readOpts = wait ? { timeout } : { wait: false }
 
   const promises = []
@@ -233,7 +216,7 @@ async function getOplogBatch(undo, core, length, weight, index, readOpts = null)
   const legacy = oplog.version <= LEGACY_OPLOG_VERSION
 
   // if this is legacy the start pointer is always 0, fix
-  if (legacy) {
+  if (legacy && !oplog.links.length) {
     for (let i = length - 1; i > 0; i--) {
       const block = await core.get(i - 1, readOpts)
       if (block === null) return null // wait: false in replay helper
@@ -241,6 +224,7 @@ async function getOplogBatch(undo, core, length, weight, index, readOpts = null)
       const oplog = encoding.decodeOplog(block)
       if (oplog.batch.end === 0) break
       head.batch.start++
+      if (oplog.links.length > 0) break // always start of batch in legacy
     }
   }
 
@@ -345,6 +329,8 @@ function getOplogCompat(data) {
   const candidates = []
 
   for (const { keys } of data.batch) {
+    if (!keys) continue
+
     for (const k of keys) {
       if (k.key[0] !== 1) continue
 

package/lib/updates.js

@@ -24,7 +24,7 @@ module.exports = class UpdateChanges {
     }
   }
 
-  finalise() {
+  finalise(fastForward) {
     // flush count at `to` - safe to read live, state is frozen for the hook
     const flushes = this.auto.system.flushes
 
@@ -33,6 +33,7 @@ module.exports = class UpdateChanges {
 
       tracked.to = this.tracking[k].head()
       tracked.flushes = flushes
+      tracked.incremental = !fastForward
     }
 
     return this.byName

package/lib/writers.js

@@ -552,15 +552,6 @@ class Writer {
       node.timestamp = timestamp = Math.max(node.timestamp, timestamp)
       node.optimistic = optimistic = optimistic || node.optimistic
 
-      if (!this.isAdded && !node.optimistic && !genesis) {
-        return null
-      }
-
-      // an optimistic writer cannot be genesis
-      if (node.optimistic && genesis) {
-        return null
-      }
-
       if (node.optimistic && !this.isAdded && b.end >= BATCH_OPTIMISTIC_SANITY) {
         return null
       }
@@ -578,6 +569,15 @@ class Writer {
         return null
       }
 
+      // an optimistic writer cannot be genesis
+      if (node.optimistic && genesis) {
+        return null
+      }
+
+      if (!this.isAdded && !node.optimistic && !genesis) {
+        return null
+      }
+
       batch.push(node)
       if (b.end === 0) return finalizeNodeBatch(batch)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",