autobee 2.0.0-rc.22 → 2.0.0-rc.25

package/index.js

@@ -22,6 +22,7 @@ const { ActiveWriters } = require('./lib/writers.js')
 const TrustedPeers = require('./lib/trusted.js')
 const ApplyView = require('./lib/apply-view.js')
 const UpdateChanges = require('./lib/updates.js')
+const migrations = require('./lib/migrations.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const INTERRUPT = new Error('Apply interrupted')
@@ -78,13 +79,16 @@ module.exports = class Autobee extends ReadyResource {
     this.writers = null
     this.bumping = 0
 
-    const fastForward = handlers.fastForward || {}
+    // fastForward: false disables all fast-forwards - both wakeup and boot
+    const fastForward = handlers.fastForward === false ? null : handlers.fastForward || {}
+
+    this._ffEnabled = fastForward !== null
 
     // oplog head to boot from: migrates or fast-forwards depending on its version
-    this.bootFrom = fastForward.boot || null
+    this.bootFrom = (fastForward && fastForward.boot) || null
 
     // conservative (default on): only fast-forward onto a head someone can serve whole
-    this._conservativeFF = fastForward.conservative !== false
+    this._conservativeFF = !fastForward || fastForward.conservative !== false
 
     this.trusted = new TrustedPeers(handlers)
 
@@ -112,6 +116,8 @@ module.exports = class Autobee extends ReadyResource {
     this._bootingAll = null
 
     this._now = handlers.now || Date.now // overridable for clock-drift tests
+    this._preapply = handlers.preapply || null
+    this._preApplied = false
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
     this._needsUpdate = false
@@ -504,9 +510,20 @@ module.exports = class Autobee extends ReadyResource {
     this.emit('error', err)
   }
 
+  // one-shot user gate: nothing applies until the host has resolved whatever
+  // state apply depends on (e.g. legacy views recorded by a migration)
+  async _runPreApply() {
+    if (this._preapply === null || this._preApplied) return
+
+    this._preApplied = true
+    await this._preapply(this.view)
+  }
+
   async _drain() {
     if (this._updating) await this._updating
 
+    await this._runPreApply()
+
     this.stats.drains++
 
     if (this.bootFrom) {
@@ -626,6 +643,8 @@ module.exports = class Autobee extends ReadyResource {
     const hints = await this._applyWakeupHints()
     if (!hints.size) return
 
+    if (!this._ffEnabled) return
+
     // a scheduled fast-forward is applied by the drain before we look again
     if (this.fastForwardTo !== null || this.fastForwarding !== null) return
     if (this._interrupting || this.closing || this.bootFrom) return
@@ -700,7 +719,7 @@ module.exports = class Autobee extends ReadyResource {
     // legacy nodes always inflate, but only indexers carry views - callers
     // that need system info must check op.views
     if (op.version < 3) {
-      op = await this._inflateLegacyOplog(buf, core, target - 1, opts)
+      op = await migrations.inflateLegacyOplog(buf, core, target - 1, opts)
     }
 
     return {
@@ -710,49 +729,6 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _inflateLegacyOplog(buf, core, seq, opts = null) {
-    const m = encoding.decodeRawOplog(buf)
-
-    const op = {
-      version: m.version,
-      timestamp: 0,
-      links: m.node.heads,
-      batch: { start: 0, end: m.node.batch - 1 },
-      views: null,
-      optimistic: !!m.optimistic,
-      value: m.node.value
-    }
-
-    if (m.digest === null || m.checkpoint === null || !m.checkpoint.system) return op
-
-    const fetches = []
-
-    fetches.push(m.digest.pointer ? core.get(seq - m.digest.pointer, opts) : buf)
-    fetches.push(
-      m.checkpoint.pointer ? core.get(seq - m.checkpoint.system.checkpointer, opts) : buf
-    )
-
-    const [digestNode, checkpointNode] = await Promise.all(fetches)
-    // a wait: false get we could not satisfy locally
-    if (digestNode === null || checkpointNode === null) return op
-
-    const { digest } = encoding.decodeRawOplog(digestNode)
-    const { checkpoint } = encoding.decodeRawOplog(checkpointNode)
-
-    if (!checkpoint.system || !checkpoint.system.checkpoint) return op
-
-    op.views = {
-      system: {
-        key: digest.key,
-        start: 0,
-        length: checkpoint.system.checkpoint.length
-      },
-      flushes: seq
-    }
-
-    return op
-  }
-
   async _update(changes) {
     this._needsUpdate = false
     this.bee.update(this._workingBee.root)

package/lib/boot.js

@@ -1,20 +1,5 @@
-const c = require('compact-encoding')
-const b4a = require('b4a')
-const crypto = require('hypercore-crypto')
-const { AutobeeEncryption } = require('autobee-encryption')
-const { decodeBlock } = require('hyperbee2/lib/encoding.js')
-
 const encoding = require('./encoding.js')
-const { assert, bail } = require('./asserts.js')
-
-const { getEncoding } = require('../encoding/spec/autobee')
-
-const SystemInfoV1 = getEncoding('@autobase-compat/info-v1')
-const SystemInfoV2 = getEncoding('@autobase-compat/info-v2')
-
-const EMPTY = b4a.alloc(0)
-const INDEX_VERSION = 1
-const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
+const { checkAutobaseMigration } = require('./migrations.js')
 
 module.exports = async function boot(
   corestore,
@@ -153,64 +138,6 @@ module.exports = async function boot(
   return result
 }
 
-async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
-  const bootRecord = await local.getUserData('autobase/boot')
-  if (!bootRecord) return null
-
-  const { key } = encoding.decodeAutobaseBootRecord(bootRecord)
-  const core = store.get({ key: key, encryption: null })
-  await core.ready()
-
-  const system = { key, length: core.length }
-
-  // setup encryption
-  const encryptionKey = await local.getUserData('autobase/encryption')
-  await AutobeeEncryption.setSystemEncryption(bootstrap.key, encryptionKey, core)
-
-  const catchup = await getCatchupHeads(core)
-  const nodes = await Promise.all(
-    catchup.map((n) => getWriterBatch(store, n, bootstrap.key, encryptionKey))
-  )
-
-  // Decode hyperbee block using hyperbee2 compat
-  const node = decodeBlock(await core.get(core.length - 1))
-  assert(node.keys.length > 0, 'bad system block')
-
-  // Decode system info from hyperbee block using autobee compat
-  const info = decodeLegacySystemInfo(node.keys[0].value)
-
-  const indexerManifests = await Promise.all(
-    info.indexers.map((idx) => getCoreManifest(store, idx.key))
-  )
-  const views = await Promise.all(info.views.map((v) => getPersistedView(store, v)))
-  const entropy =
-    info.version > 1 && info.entropy ? info.entropy : indexerManifests[0].signers[0].namespace
-
-  const mappedViews = new Map()
-  for (const name of legacyViews) {
-    const v = await findViewByName(
-      store,
-      bootstrap.key,
-      encryptionKey,
-      indexerManifests,
-      views,
-      entropy,
-      name
-    )
-
-    mappedViews.set(name, v)
-  }
-
-  await core.close()
-
-  return {
-    encryptionKey,
-    system,
-    views: mappedViews,
-    catchup: nodes
-  }
-}
-
 async function getLocalKey(bootstrap) {
   const [legacy, current] = await Promise.all([
     bootstrap.getUserData('autobase/local'),
@@ -220,121 +147,3 @@ async function getLocalKey(bootstrap) {
   if (current) return current
   return legacy
 }
-
-async function getPersistedView(store, view) {
-  const core = store.get(view.key)
-  await core.ready()
-
-  const length = core.length
-  await core.close()
-
-  return { key: view.key, length }
-}
-
-async function getCatchupHeads(core) {
-  const session = core.session({ name: 'batch' })
-  await session.ready()
-
-  const nodes = []
-  for (let i = core.length; i < session.length; i++) {
-    const node = await session.get(i, { wait: false })
-    if (!node) throw new Error('Expect nodes to exist locally')
-
-    const entry = decodeBlock(node).keys[0]
-    if (entry.key[0] !== 0x01) continue
-
-    const { key, length } = encoding.decodeSystemWriter(entry.key, entry.value)
-    if (!length) continue
-    nodes.push({ key, length })
-  }
-
-  return nodes
-}
-
-async function getWriterBatch(store, head, key, encryptionKey, nodes = []) {
-  const batch = []
-
-  const core = store.get(head.key)
-  await core.ready()
-
-  if (encryptionKey) {
-    await core.setEncryption(AutobeeEncryption.getWriterEncryption(key, encryptionKey))
-  }
-
-  let seq = head.length - 1
-  const block = await core.get(seq--, { wait: false })
-  if (!block) throw new Error('Expect writer node to exist locally')
-
-  const node = encoding.decodeOplog(block)
-
-  batch.unshift({ ...head, ...node, from: core })
-
-  while (seq >= 0) {
-    const block = await core.get(seq--, { wait: false })
-    if (!block) break
-
-    const node = encoding.decodeOplog(block)
-    if (!node.batch.end) break
-
-    batch.unshift({ key: head.key, length: seq + 2, from: core, ...node })
-  }
-
-  return batch
-}
-
-async function getCoreManifest(store, key) {
-  const core = store.get(key)
-  await core.ready()
-
-  const manifest = core.manifest
-  await core.close()
-
-  return manifest
-}
-
-function decodeLegacySystemInfo(buffer) {
-  const state = { start: 0, end: buffer.length, buffer }
-  const version = c.uint.decode(state)
-
-  state.start--
-
-  switch (version) {
-    case 1:
-      return SystemInfoV1.decode(state)
-    case 2:
-      return SystemInfoV2.decode(state)
-    default:
-      bail('Expected legacy system info')
-  }
-}
-
-function deriveNamespace(name, bootstrap, entropy, encryptionKey) {
-  const encryptionId = crypto.hash(encryptionKey || EMPTY)
-  const version = c.encode(c.uint, INDEX_VERSION)
-
-  return crypto.hash([
-    NS_SIGNER_NAMESPACE,
-    version,
-    bootstrap,
-    encryptionId,
-    entropy,
-    b4a.from(name)
-  ])
-}
-
-async function findViewByName(store, key, encryptionKey, indexerManifests, views, entropy, name) {
-  if (indexerManifests.length === 0) return null
-
-  const namespace = deriveNamespace(name, key, entropy, encryptionKey)
-
-  for (const v of views) {
-    const manifest = await getCoreManifest(store, v.key)
-    if (manifest.signers.length === 0) continue
-
-    const signer = manifest.signers[0]
-
-    if (b4a.equals(signer.namespace, namespace)) return v
-  }
-
-  return null
-}

package/lib/fast-forward.js

@@ -1,23 +1,11 @@
 const safetyCatch = require('safety-catch')
 const b4a = require('b4a')
 const Hypercore = require('hypercore')
-const c = require('compact-encoding')
-const crypto = require('hypercore-crypto')
-
-const { decodeBlock } = require('hyperbee2/lib/encoding.js')
 
 const System = require('./system.js')
-const { assert } = require('./asserts.js')
+const migrations = require('./migrations.js')
 const { LEGACY_AUTOBASE_VERSION } = require('./constants.js')
-const { getEncoding } = require('../encoding/spec/autobee')
-
-const SystemInfoV1 = getEncoding('@autobase-compat/info-v1')
-const SystemInfoV2 = getEncoding('@autobase-compat/info-v2')
 
-const INDEX_VERSION = 1
-const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
-
-const EMPTY = b4a.alloc(0)
 const DEFAULT_OP_TIMEOUT = 5_000
 const MIN_FF_GAP = 32
 
@@ -243,7 +231,25 @@ module.exports = class FastForward {
         return null
       }
 
-      return this._migrate()
+      const result = await this._migrate()
+      if (result === null || result.autobee === null) return result
+
+      // an indexer of the newest legacy generation already moved to autobee -
+      // fast-forward straight onto its announced head instead of migrating,
+      // falling back to the legacy migration if the head cannot be booted
+      try {
+        this.head = result.autobee
+        await this.system.boot(this.head, { timeout: this.timeout })
+        if (this.system.version <= LEGACY_AUTOBASE_VERSION) {
+          throw new Error('Expected an autobee system')
+        }
+      } catch (err) {
+        safetyCatch(err)
+        this.head = result.head
+        return { head: result.head, tip: null, migrate: result.migrate }
+      }
+
+      // fall through to the plain fast-forward below
     }
 
     const promises = []
@@ -280,47 +286,58 @@ module.exports = class FastForward {
   // Upgrade a legacy (pre-AUTOBEE_VERSION) system in place. The system already
   // booted above (coerced to the current struct), but the original v1/v2 struct
   // is needed to recover the indexers/views/entropy, so decode the raw block.
+  //
+  // A head may be arbitrarily old: an indexer rotation freezes the system core
+  // and moves the indexers to a successor. The frozen core cannot reveal its
+  // successor (the promotion only lands in its unsigned tail), but the
+  // indexers' newest oplog digests stamp the newest system they know - chase
+  // those so the migration lands on the newest generation instead of the head
+  // we were given.
   async _migrate() {
-    const core = this.auto.store.get({ key: this.head.key, encryption: null })
-    this.cores.push(core)
-    await core.ready()
+    let head = { key: this.head.key, length: this.head.length }
+    const seen = new Set()
 
-    // setup encryption
-    await core.setEncryption(this.auto.getSystemEncryption())
+    for (let hops = 0; hops < migrations.MAX_MIGRATE_HOPS; hops++) {
+      seen.add(b4a.toString(head.key, 'hex'))
 
-    // Decode hyperbee block using hyperbee2 compat
-    const node = decodeBlock(
-      await core.get(this.head.length - 1, {
+      const core = this.auto.store.get({ key: head.key, encryption: null })
+      this.cores.push(core)
+      await core.ready()
+
+      // setup encryption
+      await core.setEncryption(this.auto.getSystemEncryption())
+
+      // successors are resolved by key only - wait for the replicated length
+      if (!head.length) head.length = await migrations.coreLength(core, this.timeout)
+      if (!head.length || this.destroyed) return null
+
+      const info = await migrations.readLegacySystemInfo(core, head.length, {
         timeout: this.timeout,
         activeRequests: this.activeRequests
       })
-    )
-    assert(node.keys.length > 0, 'bad system block')
-
-    // Decode system info from hyperbee block using autobee compat
-    const info = decodeLegacySystemInfo(node.keys[0].value)
 
-    const indexerManifests = await Promise.all(
-      info.indexers.map((idx) => this.getCoreManifest(idx.key, idx.length))
-    )
-    const entropy =
-      info.version > 1 && info.entropy ? info.entropy : indexerManifests[0].signers[0].namespace
+      const { legacy, autobee } = await migrations.findNewestSystem(this, info)
+      if (this.destroyed) return null
 
-    const views = new Map()
-    const viewProms = []
+      if (legacy !== null && !seen.has(b4a.toString(legacy, 'hex'))) {
+        head = { key: legacy, length: 0 }
+        continue
+      }
 
-    for (const name of this.auto.legacyViews) {
-      viewProms.push(this.findViewByName(views, indexerManifests, info.views, entropy, name))
-    }
+      this.head = head
 
-    await Promise.all(viewProms)
-    if (this.destroyed) return null
+      const views = await migrations.resolveLegacyViews(this, info)
+      if (views === null || this.destroyed) return null
 
-    return {
-      head: this.head,
-      tip: null,
-      migrate: views
+      return {
+        head,
+        tip: null,
+        migrate: views,
+        autobee: autobee === null ? null : batchToHead(autobee.op.views.system)
+      }
     }
+
+    return null
   }
 
   async _resolveLength() {
@@ -329,47 +346,7 @@ module.exports = class FastForward {
 
     await core.ready()
 
-    return coreLength(core, this.timeout)
-  }
-
-  async getCoreManifest(key, length) {
-    const core = this.auto.store.get(key)
-    this.cores.push(core)
-
-    await core.ready()
-
-    try {
-      if (!core.manifest) {
-        await core.get(length - 1, {
-          timeout: this.timeout,
-          activeRequests: this.activeRequests
-        })
-      }
-    } catch {
-      return null
-    }
-
-    return core.manifest
-  }
-
-  async findViewByName(result, indexerManifests, views, entropy, name) {
-    if (indexerManifests.length === 0) return null
-
-    const namespace = deriveNamespace(name, this.auto.key, entropy, this.auto.encryptionKey)
-
-    for (const v of views) {
-      const manifest = await this.getCoreManifest(v.key, v.length)
-      if (!manifest) continue
-
-      if (manifest.signers.length === 0) continue
-
-      const signer = manifest.signers[0]
-
-      if (b4a.equals(signer.namespace, namespace)) {
-        result.set(name, v)
-        return
-      }
-    }
+    return migrations.coreLength(core, this.timeout)
   }
 
   // cancels every read in flight and fails any that come after
@@ -391,42 +368,3 @@ function batchToHead(b) {
     length: b.start + b.length
   }
 }
-
-function decodeLegacySystemInfo(buffer) {
-  const state = { start: 0, end: buffer.length, buffer }
-  const version = c.uint.decode(state)
-
-  state.start--
-
-  switch (version) {
-    case 1:
-      return SystemInfoV1.decode(state)
-    case 2:
-      return SystemInfoV2.decode(state)
-    default:
-      return { version }
-  }
-}
-
-function coreLength(core, timeout) {
-  if (core.length) return core.length
-
-  return new Promise((resolve) => {
-    core.on('append', () => resolve(core.length))
-    setTimeout(resolve, timeout, 0)
-  })
-}
-
-function deriveNamespace(name, bootstrap, entropy, encryptionKey) {
-  const encryptionId = crypto.hash(encryptionKey || EMPTY)
-  const version = c.encode(c.uint, INDEX_VERSION)
-
-  return crypto.hash([
-    NS_SIGNER_NAMESPACE,
-    version,
-    bootstrap,
-    encryptionId,
-    entropy,
-    b4a.from(name)
-  ])
-}

package/lib/migrations.js

@@ -0,0 +1,421 @@
+const c = require('compact-encoding')
+const b4a = require('b4a')
+const crypto = require('hypercore-crypto')
+const { AutobeeEncryption } = require('autobee-encryption')
+const { decodeBlock } = require('hyperbee2/lib/encoding.js')
+
+const encoding = require('./encoding.js')
+const { assert, bail } = require('./asserts.js')
+const { LEGACY_OPLOG_VERSION } = require('./constants.js')
+
+const { getEncoding } = require('../encoding/spec/autobee')
+
+const SystemInfoV1 = getEncoding('@autobase-compat/info-v1')
+const SystemInfoV2 = getEncoding('@autobase-compat/info-v2')
+
+const EMPTY = b4a.alloc(0)
+const INDEX_VERSION = 1
+const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
+
+// indexer rotations chain legacy systems - how many generations we chase
+const MAX_MIGRATE_HOPS = 64
+
+module.exports = {
+  MAX_MIGRATE_HOPS,
+  checkAutobaseMigration,
+  readLegacySystemInfo,
+  findNewestSystem,
+  resolveLegacyViews,
+  inflateLegacyOplog,
+  decodeLegacySystemInfo,
+  deriveNamespace,
+  coreLength
+}
+
+// boot-time migration of local legacy autobase storage
+async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
+  const bootRecord = await local.getUserData('autobase/boot')
+  if (!bootRecord) return null
+
+  const { key } = encoding.decodeAutobaseBootRecord(bootRecord)
+  const core = store.get({ key: key, encryption: null })
+  await core.ready()
+
+  const system = { key, length: core.length }
+
+  // setup encryption
+  const encryptionKey = await local.getUserData('autobase/encryption')
+  await AutobeeEncryption.setSystemEncryption(bootstrap.key, encryptionKey, core)
+
+  const catchup = await getCatchupHeads(core)
+  const nodes = await Promise.all(
+    catchup.map((n) => getWriterBatch(store, n, bootstrap.key, encryptionKey))
+  )
+
+  const info = await readLegacySystemInfo(core, core.length)
+
+  const indexerManifests = await Promise.all(
+    info.indexers.map((idx) => storeCoreManifest(store, idx.key))
+  )
+  const views = await Promise.all(info.views.map((v) => getPersistedView(store, v)))
+  const entropy = legacyEntropy(info, indexerManifests)
+
+  const getManifest = (key) => storeCoreManifest(store, key)
+
+  const mappedViews = new Map()
+  for (const name of legacyViews) {
+    const v = await findViewByName(
+      getManifest,
+      bootstrap.key,
+      encryptionKey,
+      indexerManifests,
+      views,
+      entropy,
+      name
+    )
+
+    mappedViews.set(name, v)
+  }
+
+  await core.close()
+
+  return {
+    encryptionKey,
+    system,
+    views: mappedViews,
+    catchup: nodes
+  }
+}
+
+// decode the legacy system info recorded in the system core's block at length
+async function readLegacySystemInfo(core, length, opts = null) {
+  // Decode hyperbee block using hyperbee2 compat
+  const node = decodeBlock(await core.get(length - 1, opts))
+  assert(node.keys.length > 0, 'bad system block')
+
+  // Decode system info from hyperbee block using autobee compat
+  return decodeLegacySystemInfo(node.keys[0].value)
+}
+
+// read each indexer's newest legacy oplog stamp - its digest/checkpoint mark
+// the newest legacy system that indexer knows. take the majority vote. an
+// indexer that already moved to autobee announces its live autobee state
+// instead - keep the newest of those as a direct fast-forward candidate and
+// bisect down to its buried legacy stamps so its vote still counts
+async function findNewestSystem(ff, info) {
+  const tally = new Map()
+
+  let autobee = null
+
+  for (const idx of info.indexers) {
+    const core = ff.auto.openCore(idx.key)
+    ff.cores.push(core)
+    await core.ready()
+
+    const length = await coreLength(core, ff.timeout)
+    if (!length || ff.destroyed) continue
+
+    let oplog = await flushHead(ff.auto, { key: idx.key, length }, { timeout: ff.timeout })
+    if (oplog === null || !oplog.op.views) continue
+
+    if (oplog.op.version > LEGACY_OPLOG_VERSION) {
+      if (autobee === null || oplog.op.views.flushes > autobee.op.views.flushes) {
+        autobee = oplog
+      }
+
+      const boundary = await findLegacyBoundary(ff, core, idx.length, oplog.length - 1)
+      if (!boundary || ff.destroyed) continue
+
+      oplog = await flushHead(ff.auto, { key: idx.key, length: boundary }, { timeout: ff.timeout })
+      if (oplog === null || !oplog.op.views) continue
+      if (oplog.op.version > LEGACY_OPLOG_VERSION) continue
+    }
+
+    const sys = oplog.op.views.system
+    if (!sys || !sys.key) continue
+
+    const hex = b4a.toString(sys.key, 'hex')
+    const t = tally.get(hex) || { key: sys.key, votes: 0, length: sys.length }
+
+    t.votes++
+    if (sys.length > t.length) t.length = sys.length
+    tally.set(hex, t)
+  }
+
+  const majority = (info.indexers.length >> 1) + 1
+
+  let best = null
+  for (const t of tally.values()) {
+    if (t.votes >= majority && (best === null || t.length > best.length)) best = t
+  }
+
+  return { legacy: best === null ? null : best.key, autobee }
+}
+
+// map the legacy views recorded in the system info by name
+async function resolveLegacyViews(ff, info) {
+  const indexerManifests = await Promise.all(
+    info.indexers.map((idx) => ffCoreManifest(ff, idx.key, idx.length))
+  )
+  if (ff.destroyed || indexerManifests.some((m) => m === null)) return null
+
+  const entropy = legacyEntropy(info, indexerManifests)
+
+  const getManifest = (key, length) => ffCoreManifest(ff, key, length)
+
+  const views = new Map()
+
+  await Promise.all(
+    ff.auto.legacyViews.map(async (name) => {
+      const v = await findViewByName(
+        getManifest,
+        ff.auto.key,
+        ff.auto.encryptionKey,
+        indexerManifests,
+        info.views,
+        entropy,
+        name
+      )
+
+      if (v) views.set(name, v)
+    })
+  )
+
+  if (ff.destroyed) return null
+
+  return views
+}
+
+// legacy nodes always inflate, but only indexers carry the digest/checkpoint
+// stamps that synthesise system info - callers must check op.views
+async function inflateLegacyOplog(buf, core, seq, opts = null) {
+  const m = encoding.decodeRawOplog(buf)
+
+  const op = {
+    version: m.version,
+    timestamp: 0,
+    links: m.node.heads,
+    batch: { start: 0, end: m.node.batch - 1 },
+    views: null,
+    optimistic: !!m.optimistic,
+    value: m.node.value
+  }
+
+  if (m.digest === null || m.checkpoint === null || !m.checkpoint.system) return op
+
+  const fetches = []
+
+  fetches.push(m.digest.pointer ? core.get(seq - m.digest.pointer, opts) : buf)
+  fetches.push(m.checkpoint.pointer ? core.get(seq - m.checkpoint.system.checkpointer, opts) : buf)
+
+  const [digestNode, checkpointNode] = await Promise.all(fetches)
+  // a wait: false get we could not satisfy locally
+  if (digestNode === null || checkpointNode === null) return op
+
+  const { digest } = encoding.decodeRawOplog(digestNode)
+  const { checkpoint } = encoding.decodeRawOplog(checkpointNode)
+
+  if (!checkpoint.system || !checkpoint.system.checkpoint) return op
+
+  op.views = {
+    system: {
+      key: digest.key,
+      start: 0,
+      length: checkpoint.system.checkpoint.length
+    },
+    flushes: seq
+  }
+
+  return op
+}
+
+function decodeLegacySystemInfo(buffer) {
+  const state = { start: 0, end: buffer.length, buffer }
+  const version = c.uint.decode(state)
+
+  state.start--
+
+  switch (version) {
+    case 1:
+      return SystemInfoV1.decode(state)
+    case 2:
+      return SystemInfoV2.decode(state)
+    default:
+      bail('Expected legacy system info')
+  }
+}
+
+function deriveNamespace(name, bootstrap, entropy, encryptionKey) {
+  const encryptionId = crypto.hash(encryptionKey || EMPTY)
+  const version = c.encode(c.uint, INDEX_VERSION)
+
+  return crypto.hash([
+    NS_SIGNER_NAMESPACE,
+    version,
+    bootstrap,
+    encryptionId,
+    entropy,
+    b4a.from(name)
+  ])
+}
+
+function coreLength(core, timeout) {
+  if (core.length) return core.length
+
+  return new Promise((resolve) => {
+    core.on('append', () => resolve(core.length))
+    setTimeout(resolve, timeout, 0)
+  })
+}
+
+// a writer's oplog is a legacy prefix followed by autobee-format nodes -
+// bisect for the boundary. lo is a length known to be legacy (the legacy
+// system's recorded length for this writer), hi a seq known to be autobee.
+// returns the legacy prefix length
+async function findLegacyBoundary(ff, core, lo, hi) {
+  while (lo < hi) {
+    const mid = (lo + hi) >>> 1
+
+    const block = await core.get(mid, {
+      timeout: ff.timeout,
+      activeRequests: ff.activeRequests
+    })
+
+    if (encoding.decodeRawOplog(block).version > LEGACY_OPLOG_VERSION) hi = mid
+    else lo = mid + 1
+  }
+
+  return lo
+}
+
+function legacyEntropy(info, indexerManifests) {
+  return info.version > 1 && info.entropy ? info.entropy : indexerManifests[0].signers[0].namespace
+}
+
+async function findViewByName(
+  getManifest,
+  bootstrap,
+  encryptionKey,
+  indexerManifests,
+  views,
+  entropy,
+  name
+) {
+  if (indexerManifests.length === 0) return null
+
+  const namespace = deriveNamespace(name, bootstrap, entropy, encryptionKey)
+
+  for (const v of views) {
+    const manifest = await getManifest(v.key, v.length)
+    if (!manifest) continue
+
+    if (manifest.signers.length === 0) continue
+
+    const signer = manifest.signers[0]
+
+    if (b4a.equals(signer.namespace, namespace)) return v
+  }
+
+  return null
+}
+
+// request-tracked manifest read over replication
+async function ffCoreManifest(ff, key, length) {
+  const core = ff.auto.store.get(key)
+  ff.cores.push(core)
+
+  await core.ready()
+
+  try {
+    if (!core.manifest) {
+      await core.get(length - 1, {
+        timeout: ff.timeout,
+        activeRequests: ff.activeRequests
+      })
+    }
+  } catch {
+    return null
+  }
+
+  return core.manifest
+}
+
+// local-storage manifest read
+async function storeCoreManifest(store, key) {
+  const core = store.get(key)
+  await core.ready()
+
+  const manifest = core.manifest
+  await core.close()
+
+  return manifest
+}
+
+async function getPersistedView(store, view) {
+  const core = store.get(view.key)
+  await core.ready()
+
+  const length = core.length
+  await core.close()
+
+  return { key: view.key, length }
+}
+
+async function getCatchupHeads(core) {
+  const session = core.session({ name: 'batch' })
+  await session.ready()
+
+  const nodes = []
+  for (let i = core.length; i < session.length; i++) {
+    const node = await session.get(i, { wait: false })
+    if (!node) throw new Error('Expect nodes to exist locally')
+
+    const entry = decodeBlock(node).keys[0]
+    if (entry.key[0] !== 0x01) continue
+
+    const { key, length } = encoding.decodeSystemWriter(entry.key, entry.value)
+    if (!length) continue
+    nodes.push({ key, length })
+  }
+
+  await session.close()
+
+  return nodes
+}
+
+async function getWriterBatch(store, head, key, encryptionKey, nodes = []) {
+  const batch = []
+
+  const core = store.get(head.key)
+  await core.ready()
+
+  if (encryptionKey) {
+    await core.setEncryption(AutobeeEncryption.getWriterEncryption(key, encryptionKey))
+  }
+
+  let seq = head.length - 1
+  const block = await core.get(seq--, { wait: false })
+  if (!block) throw new Error('Expect writer node to exist locally')
+
+  const node = encoding.decodeOplog(block)
+
+  batch.unshift({ ...head, ...node, from: core })
+
+  while (seq >= 0) {
+    const block = await core.get(seq--, { wait: false })
+    if (!block) break
+
+    const node = encoding.decodeOplog(block)
+    if (!node.batch.end) break
+
+    batch.unshift({ key: head.key, length: seq + 2, from: core, ...node })
+  }
+
+  return batch
+}
+
+// lazy to avoid the require cycle - fast-forward imports this module
+function flushHead(auto, head, opts) {
+  const FastForward = require('./fast-forward.js')
+  return FastForward.flushHead(auto, head, opts)
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.22",
+  "version": "2.0.0-rc.25",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",