autobee 2.11.3 → 2.11.5

package/index.js

@@ -29,6 +29,7 @@ const migrations = require('./lib/migrations.js')
 const EMPTY_HEAD = { length: 0, key: null }
 const DEFAULT_ACK_THRESHOLD = 32
 const BOOT_NETWORK_TIMEOUT = 5000
+const REINDEX_PREFETCH = 4096
 const INTERRUPT = new Error('Apply interrupted')
 
 module.exports = class Autobee extends ReadyResource {
@@ -150,6 +151,7 @@ module.exports = class Autobee extends ReadyResource {
     this._preapply = handlers.preapply || null
     this._preApplied = false
     this._reindexed = false
+    this._strictReindex = !!handlers.strictReindex
     this._warmup = handlers.warmup || null
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
@@ -689,10 +691,14 @@ module.exports = class Autobee extends ReadyResource {
     const latest = await this.writers.getLatestLocalOplog()
     const views = latest ? latest.views : null
 
+    const from = this._workingBee.head()
     await this._reindexBee(this._workingBee, views ? views.view : null)
     this.bee.move(this._workingBee.head())
 
-    await this._reindexBee(this.system.bee, views ? views.system : null)
+    const map = await migrations.mapReindexedView(this, from, REINDEX_PREFETCH)
+    await this._reindexBee(this.system.bee, views ? views.system : null, map)
+
+    await this.system.reset()
 
     this._reindexed = true
   }
@@ -707,7 +713,7 @@ module.exports = class Autobee extends ReadyResource {
     return !(await this._shouldReindex(view.key))
   }
 
-  async _reindexBee(bee, flushed) {
+  async _reindexBee(bee, flushed, map = null) {
     const head = bee.head()
     const local = bee.context.local
     if (head === null) return
@@ -718,7 +724,10 @@ module.exports = class Autobee extends ReadyResource {
       return
     }
 
-    await bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
+    await bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)), {
+      prefetch: REINDEX_PREFETCH,
+      map
+    })
     bee.move({ key: local.key, length: local.length })
   }
 
@@ -733,7 +742,8 @@ module.exports = class Autobee extends ReadyResource {
       }
 
       if (core.manifest === null) return unknown
-      return core.manifest.version > 1 && core.manifest.version < DEFAULT_MANIFEST_VERSION
+      if (core.manifest.version >= DEFAULT_MANIFEST_VERSION) return false
+      return this._strictReindex || core.manifest.version > 1
     } finally {
       await core.close()
     }
@@ -1655,17 +1665,16 @@ module.exports = class Autobee extends ReadyResource {
       const hints = await this._applyWakeupHints()
       if (hints.length) this._queueFastForward(hints)
 
-      const remaining = deadline - Date.now()
-      if (remaining <= 0 || this.fastForwardTo !== null) return
-
+      // the search bounds its own reads, so it always settles
       if (this._ffSearching !== null) {
-        await Promise.race([this._ffSearching, this._bumpSignal.wait(remaining)])
-        this._bumpSignal.notify()
-        if (this._ffSearching === null) return
-        continue
+        await this._ffSearching
+        return
       }
 
-      if (hints.length) return
+      if (hints.length || this.fastForwardTo !== null) return
+
+      const remaining = deadline - Date.now()
+      if (remaining <= 0) return
 
       await this._bumpSignal.wait(remaining)
     }

package/lib/migrations.js

@@ -6,6 +6,7 @@ const { decodeBlock } = require('hyperbee2/lib/encoding.js')
 const { AutobeeEncryption } = require('./encryption.js')
 const encoding = require('./encoding.js')
 const { assert, bail } = require('./asserts.js')
+const { LEGACY_AUTOBASE_VERSION } = require('./constants.js')
 
 const { getEncoding } = require('../encoding/spec/autobee')
 
@@ -15,6 +16,7 @@ const SystemInfoV2 = getEncoding('@autobase-compat/info-v2')
 const EMPTY = b4a.alloc(0)
 const INDEX_VERSION = 1
 const INFO = b4a.from([0x00, 0x00, 0x69, 0x6e, 0x66, 0x6f])
+const INFO_V3 = b4a.from([0])
 const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
 
 module.exports = {
@@ -23,6 +25,7 @@ module.exports = {
   checkAutobaseMigration,
   readLegacySystemInfo,
   inflateLegacyOplog,
+  mapReindexedView,
   coreLength
 }
 
@@ -381,3 +384,50 @@ async function viewsReached(store, info) {
   }
   return true
 }
+
+// the reindex copies the view batch for batch, so old and new heads line up
+async function mapReindexedView(auto, from, prefetch) {
+  const bee = auto._workingBee
+  const to = bee.head()
+
+  if (from === null || to === null || b4a.equals(from.key, to.key)) return null
+
+  const olds = []
+  for await (const { head } of bee.createChangesStream({ head: from, prefetch })) {
+    if (!(await auto._shouldReindex(head.key))) break
+    olds.push(head)
+  }
+
+  const news = []
+  for await (const { head } of bee.createChangesStream({ head: to })) {
+    if (news.length === olds.length) break
+    news.push(head)
+  }
+
+  if (news.length !== olds.length) throw new Error('Reindexed view is missing changes')
+
+  const moved = new Map()
+
+  for (let i = 0; i < olds.length; i++) {
+    moved.set(headId(olds[i]), news[i])
+    moved.set(headId({ key: olds[i].key, length: 0 }), { key: to.key, length: 0 })
+  }
+
+  return function (key, value) {
+    if (!b4a.equals(key, INFO_V3) && !b4a.equals(key, INFO)) return null
+
+    const info = encoding.decodeSystemInfo(value)
+    if (info.version <= LEGACY_AUTOBASE_VERSION) return null
+    if (!info.view || !info.view.key) return null
+
+    const view = moved.get(headId(info.view))
+    if (!view) return null
+
+    info.view = view
+    return encoding.encodeSystemInfo(info)
+  }
+}
+
+function headId(head) {
+  return b4a.toString(head.key, 'hex') + ':' + head.length
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.11.3",
+  "version": "2.11.5",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -26,7 +26,7 @@
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
     "compact-encoding": "^3.4.0",
-    "hyperbee2": "^2.17.1",
+    "hyperbee2": "^2.18.0",
     "hypercore": "^11.37.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.26.2",