autobee 2.0.0-rc.26 → 2.0.0-rc.28

package/index.js

@@ -177,6 +177,7 @@ module.exports = class Autobee extends ReadyResource {
     this._bootingAll.catch(safetyCatch)
 
     await this.bee.ready()
+    await this._workingBee.ready()
     await this._bootingState
 
     this._localSystemStart = this.system.bee.context.local.length
@@ -1042,12 +1043,7 @@ module.exports = class Autobee extends ReadyResource {
 
     this.writers.attest(witnessed)
 
-    for (const { key, added, isAnchor } of changed) {
-      if (isAnchor) {
-        // no Writer/ActiveWriters tracking, but still wake anything linked to it
-        this.writers.triggers.trigger(b4a.toString(key, 'hex'), 1)
-        continue
-      }
+    for (const { key, added } of changed) {
       if (added) await this.writers.add(key)
       else await this.writers.remove(key)
     }
@@ -1186,7 +1182,7 @@ module.exports = class Autobee extends ReadyResource {
   // legacy: ungated boot straight onto a system head
   async _bootFromSystem(system) {
     try {
-      const ff = new FastForward(this, system, null, { timeout: FastForward.DEFAULT_TIMEOUT })
+      const ff = new FastForward(this, system, { timeout: FastForward.DEFAULT_TIMEOUT })
       return await this._runFastForward(ff)
     } catch (err) {
       safetyCatch(err)
@@ -1326,7 +1322,7 @@ module.exports = class Autobee extends ReadyResource {
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
-    const { head, tip, migrate } = this.fastForwardTo
+    const { head, migrate } = this.fastForwardTo
 
     const from = this.system.bee.head()
     const to = head
@@ -1366,28 +1362,6 @@ module.exports = class Autobee extends ReadyResource {
     this.stats.fastForwards++
     this.emit('move-to', to, from)
     this.ff.resolve({ to, from })
-
-    // tip is null during boot
-    if (!tip) return
-
-    try {
-      await this._reapply(tip)
-    } catch (err) {
-      throw err
-    }
-  }
-
-  async _reapply({ system, verified }) {
-    const changes = this._hasUpdate ? new UpdateChanges(this) : null
-    if (changes) changes.track()
-
-    const sys = this.system.bee.checkout(system)
-    const t = await topo.rollback(this, sys, verified)
-    await sys.close()
-
-    if (t !== null) await this.applyBacklog(t.tip)
-
-    return this._update(changes)
   }
 
   replay() {

package/lib/boot.js

@@ -1,3 +1,4 @@
+const b4a = require('b4a')
 const encoding = require('./encoding.js')
 const { checkAutobaseMigration } = require('./migrations.js')
 
@@ -94,7 +95,7 @@ module.exports = async function boot(
     await result.bootstrap.setUserData('autobee/local', result.local.key)
     await result.local.setUserData('referrer', result.key)
 
-    result.migration = await checkAutobaseMigration(
+    result.migration = await migrateWithFallback(
       corestore,
       result.local,
       result.bootstrap,
@@ -138,6 +139,23 @@ module.exports = async function boot(
   return result
 }
 
+// the bootstrap pointer names the rotated-to writer, whose legacy boot record
+// can be unusable. the namespace local still carries the pre-rotation record,
+// which points at the last system core that was actually signed
+async function migrateWithFallback(corestore, local, bootstrap, legacyViews) {
+  try {
+    return await checkAutobaseMigration(corestore, local, bootstrap, legacyViews)
+  } catch (err) {
+    const prev = corestore.get({ name: 'local', active: false })
+    await prev.ready()
+
+    if (b4a.equals(prev.key, local.key)) throw err
+    if (!(await prev.getUserData('autobase/boot'))) throw err
+
+    return checkAutobaseMigration(corestore, prev, bootstrap, legacyViews)
+  }
+}
+
 async function getLocalKey(bootstrap) {
   const [legacy, current] = await Promise.all([
     bootstrap.getUserData('autobase/local'),

package/lib/fast-forward.js

@@ -14,7 +14,7 @@ const MIN_FF_GAP = 32
 const MAX_FLUSH_SEARCH = 64
 
 module.exports = class FastForward {
-  constructor(auto, head, tip, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
+  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT } = {}) {
     this.auto = auto
 
     this.activeRequests = []
@@ -30,7 +30,6 @@ module.exports = class FastForward {
     // length -1 marks an uninitialised head: we know the key but not the length
     this.head = { key: head.key, length: head.length ?? 0 }
     this.timeout = timeout
-    this.tip = tip || null
     this.destroyed = false
     this.running = null
     this.failed = false
@@ -68,15 +67,7 @@ module.exports = class FastForward {
       return null
     }
 
-    const tip = {
-      system: batchToHead(oplog.op.views.system),
-      verified: {
-        op: { key: verified.key, length: verified.length },
-        flushes: verified.op.views.flushes
-      }
-    }
-
-    return new FastForward(auto, batchToHead(verified.op.views.system), tip)
+    return new FastForward(auto, batchToHead(verified.op.views.system))
   }
 
   static async fromHeads(
@@ -246,7 +237,7 @@ module.exports = class FastForward {
       } catch (err) {
         safetyCatch(err)
         this.head = result.head
-        return { head: result.head, tip: null, migrate: result.migrate }
+        return { head: result.head, migrate: result.migrate }
       }
 
       // fall through to the plain fast-forward below
@@ -278,7 +269,6 @@ module.exports = class FastForward {
 
     return {
       head: this.head,
-      tip: this.tip,
       migrate: null
     }
   }
@@ -331,7 +321,6 @@ module.exports = class FastForward {
 
       return {
         head,
-        tip: null,
         migrate: views,
         autobee: autobee === null ? null : batchToHead(autobee.op.views.system)
       }

package/lib/migrations.js

@@ -37,23 +37,36 @@ async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
   const bootRecord = await local.getUserData('autobase/boot')
   if (!bootRecord) return null
 
-  const { key } = encoding.decodeAutobaseBootRecord(bootRecord)
+  const { key, systemLength } = encoding.decodeAutobaseBootRecord(bootRecord)
   const core = store.get({ key: key, encryption: null })
   await core.ready()
 
-  const system = { key, length: core.length }
+  const length = await findLocalSystemLength(core, systemLength)
+  assert(length > 0, 'Expected system block to exist locally')
+
+  const system = { key, length }
 
   // setup encryption
   const encryptionKey = await local.getUserData('autobase/encryption')
   await AutobeeEncryption.setSystemEncryption(bootstrap.key, encryptionKey, core)
 
-  const catchup = await getCatchupHeads(core)
+  const session = core.session({ name: 'batch' })
+  await session.ready()
+
+  let catchup
+
+  try {
+    catchup = await getCatchupHeads(session, length)
+  } finally {
+    await session.close()
+  }
+
+  const info = await readLegacySystemInfo(core, length, { wait: false })
+
   const nodes = await Promise.all(
     catchup.map((n) => getWriterBatch(store, n, bootstrap.key, encryptionKey))
   )
 
-  const info = await readLegacySystemInfo(core, core.length)
-
   const indexerManifests = await Promise.all(
     info.indexers.map((idx) => storeCoreManifest(store, idx.key))
   )
@@ -89,8 +102,11 @@ async function checkAutobaseMigration(store, local, bootstrap, legacyViews) {
 
 // decode the legacy system info recorded in the system core's block at length
 async function readLegacySystemInfo(core, length, opts = null) {
+  const block = await core.get(length - 1, opts)
+  assert(block !== null, 'Expected system block to exist locally')
+
   // Decode hyperbee block using hyperbee2 compat
-  const node = decodeBlock(await core.get(length - 1, opts))
+  const node = decodeBlock(block)
   assert(node.keys.length > 0, 'bad system block')
 
   // Decode system info from hyperbee block using autobee compat
@@ -259,6 +275,15 @@ function deriveNamespace(name, bootstrap, entropy, encryptionKey) {
   ])
 }
 
+// the record's length can run past the signed core into the local batch
+// session, and the signed core can carry verified-but-undownloaded blocks -
+// scan back for the newest block the signed core actually has locally
+async function findLocalSystemLength(core, systemLength) {
+  let length = Math.min(core.length, systemLength)
+  while (length > 0 && !(await core.has(length - 1))) length--
+  return length
+}
+
 function coreLength(core, timeout) {
   if (core.length) return core.length
 
@@ -355,18 +380,18 @@ async function getPersistedView(store, view) {
   const core = store.get(view.key)
   await core.ready()
 
-  const length = core.length
+  // core.length can run past what was downloaded - the system's recorded
+  // length is the last one we know is local
+  const length = Math.min(core.length, view.length)
   await core.close()
 
   return { key: view.key, length }
 }
 
-async function getCatchupHeads(core) {
-  const session = core.session({ name: 'batch' })
-  await session.ready()
-
+async function getCatchupHeads(session, from) {
   const nodes = []
-  for (let i = core.length; i < session.length; i++) {
+  // block 0 is the bee header, never a node
+  for (let i = Math.max(from, 1); i < session.length; i++) {
     const node = await session.get(i, { wait: false })
     if (!node) throw new Error('Expect nodes to exist locally')
 
@@ -378,8 +403,6 @@ async function getCatchupHeads(core) {
     nodes.push({ key, length })
   }
 
-  await session.close()
-
   return nodes
 }
 

package/lib/system.js

@@ -85,8 +85,8 @@ module.exports = class Systembee {
     this.update(key, length, -1, 0, false, false, true, false, false, -1, false)
   }
 
-  addAnchor(key, { length = 1 } = {}) {
-    this.update(key, length, -1, -1, false, true, false, false, true, -1, false)
+  addAnchor(key) {
+    this.update(key, 0, -1, -1, false, true, false, false, true, -1, false)
   }
 
   isGenesis() {

package/lib/topo.js

@@ -1,7 +1,8 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
-const { createNode } = require('./writers.js')
+const { createNode, finalizeNodeBatch } = require('./writers.js')
 const asserts = require('./asserts.js')
+const { LEGACY_OPLOG_VERSION } = require('./constants.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
@@ -9,7 +10,6 @@ const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 const DEFAULT_OP_TIMEOUT = 5000
 
 exports.sort = sort
-exports.rollback = rollback
 exports.replay = replay
 exports.getMostRecentHead = getMostRecentHead
 
@@ -48,46 +48,6 @@ async function getMostRecentHead(auto, system) {
   return null
 }
 
-async function rollback(auto, system, verified) {
-  if (!(await isApplied(system, verified.op))) return null
-
-  const tip = []
-  const ch = system.createChangesStream()
-
-  let found = false
-
-  for await (const data of ch) {
-    const { ref, info } = getOplog(data)
-
-    if (sameNode(ref.oplog, verified.op)) {
-      // stale views.flushes - the verified head describes a prefix this system
-      // does not share, so nothing above it is ours to reapply
-      if (info.flushes === verified.flushes) found = true
-      break
-    }
-
-    // walking back, a writer's lengths only decrease - once this writer is
-    // behind the verified head the marker cannot be further down
-    if (b4a.equals(ref.oplog.key, verified.op.key) && ref.oplog.length < verified.op.length) {
-      break
-    }
-
-    tip.push(ref)
-  }
-
-  if (!found) return null
-
-  const inflated = await inflate(auto, tip.reverse())
-
-  // @todo: verify sort
-
-  return {
-    undo: null,
-    view: null,
-    tip: inflated.map(getBatch)
-  }
-}
-
 // nodes we cannot resolve are returned as null rather than dropped, so callers
 // can tell an incomplete replay from a short one
 async function replay(auto) {
@@ -108,14 +68,6 @@ async function replay(auto) {
   return inflated.flatMap((b) => (b === null ? [null] : b.batch))
 }
 
-async function isApplied(system, node) {
-  const entry = await system.get(encoding.encodeSystemWriterKey(node.key))
-  if (entry === null) return false
-
-  const info = encoding.decodeSystemWriter(entry.key, entry.value)
-  return info.length >= node.length
-}
-
 async function sort(auto, batch) {
   const node = batch[0]
   const ch = auto.system.bee.createChangesStream()
@@ -124,7 +76,7 @@ async function sort(auto, batch) {
   for await (const data of ch) {
     if (isLegacyNode(data)) break
 
-    const { ref, info } = getOplog(data)
+    const { ref } = getOplog(data)
     const link = {
       key: ref.oplog.key,
       length: ref.oplog.length,
@@ -275,8 +227,20 @@ async function getOplogBatch(undo, core, length, weight, index, opts = null) {
   const seq = length - 1
   const block = await core.get(seq, opts)
   if (block === null) return null
+
   const oplog = encoding.decodeOplog(block)
   const head = createNode(core, length, weight, oplog)
+  const legacy = oplog.version <= LEGACY_OPLOG_VERSION
+
+  // if this is legacy the start pointer is always 0, fix
+  if (legacy) {
+    for (let i = length - 1; i > 0; i--) {
+      const block = await core.get(i - 1, opts)
+      const oplog = encoding.decodeOplog(block)
+      if (oplog.batch.end === 0) break
+      head.batch.start++
+    }
+  }
 
   const batch = []
   const result = { undo, batch, index }
@@ -304,6 +268,8 @@ async function getOplogBatch(undo, core, length, weight, index, opts = null) {
   }
 
   batch.push(head)
+  finalizeNodeBatch(batch)
+
   return result
 }
 
@@ -433,11 +399,3 @@ function isLegacyNode(data) {
   }
   return false
 }
-
-function sameNode(a, b) {
-  return b4a.equals(a.key, b.key) && a.length === b.length
-}
-
-function getBatch(b) {
-  return b.batch
-}

package/lib/writers.js

@@ -1,6 +1,7 @@
 const b4a = require('b4a')
 const encoding = require('./encoding.js')
 const Triggers = require('./triggers.js')
+const { LEGACY_OPLOG_VERSION } = require('./constants.js')
 const { signAttestation, resolveWeight } = require('./witness.js')
 const { WriterEncryption } = require('autobee-encryption')
 
@@ -62,9 +63,6 @@ class ActiveWriters {
 
     let added = false
     if (!w) {
-      // anchors are local-only and never replicated - nothing to fetch
-      const info = await this.auto.system.get(key, { unflushed: true })
-      if (info && info.isAnchor) return
       w = await this.add(key)
       if (!w) return
       added = true
@@ -453,7 +451,9 @@ class Writer {
   async update() {
     const info = await this.system.get(this.core.key)
 
-    if (info && info.isAnchor && this !== this.writers.localWriter) {
+    // an anchor core is prologue-only, so once its single node has applied
+    // nothing more can ever come - retire the writer
+    if (info && info.isAnchor && info.length > 0 && this !== this.writers.localWriter) {
       await this.detachAndClose()
       return info.length
     }
@@ -572,7 +572,7 @@ class Writer {
       }
 
       batch.push(node)
-      if (b.end === 0) return batch
+      if (b.end === 0) return finalizeNodeBatch(batch)
 
       // TODO: mark as dead
       if (batch.length >= BATCH_WRITER_SANITY) return null
@@ -763,6 +763,7 @@ function sortsBefore(a, b) {
 exports.ActiveWriters = ActiveWriters
 exports.Writer = Writer
 exports.createNode = createNode
+exports.finalizeNodeBatch = finalizeNodeBatch
 
 function createNode(core, length, weight, oplog) {
   return {
@@ -784,4 +785,14 @@ function createNode(core, length, weight, oplog) {
   }
 }
 
+function finalizeNodeBatch(batch) {
+  if (batch.length === 0 || batch[0].version > LEGACY_OPLOG_VERSION) return batch
+
+  for (let i = 0; i < batch.length; i++) {
+    batch[i].batch.start = i
+  }
+
+  return batch
+}
+
 function noop() {}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.26",
+  "version": "2.0.0-rc.28",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",