autobee 2.10.0 → 2.11.0

package/encoding/legacy.js

@@ -215,7 +215,6 @@ const OplogMessageV1 = {
     throw new Error('Encoding not supported')
   },
   decode(state) {
-    c.uint.decode(state) // version
     const maxSupportedVersion = c.uint.decode(state)
 
     const flags = c.uint.decode(state)
@@ -246,7 +245,6 @@ const OplogMessageV0 = {
     throw new Error('Encoding not supported')
   },
   decode(state) {
-    c.uint.decode(state) // version
     const flags = c.uint.decode(state)
 
     const isCheckpointer = (flags & 1) !== 0
@@ -276,6 +274,7 @@ const SystemWriterV0 = {
     throw new Error('Encoding not supported')
   },
   decode(state) {
+    state.start-- // Undo the hack to nest this in a versioned struct w/o a version
     const flags = c.uint.decode(state)
 
     return {
@@ -328,7 +327,9 @@ function oplogLegacyMap(m) {
     witness: null,
     approvals: null,
     optimistic: m.optimistic,
-    value: m.value
+    value: m.value,
+    weightHint: 0,
+    drift: 0
   }
 }
 

package/encoding/spec/autobee/index.js

@@ -108,13 +108,12 @@ const encoding4 = {
     if (m.recoveries) c.uint.encode(state, m.recoveries)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.fixed32.decode(state)
     const r1 = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       key: r0,
       systemLength: r1,
       indexersUpdated: (flags & 1) !== 0,
@@ -159,18 +158,18 @@ const encoding5 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0: {
         const decoded = encoding3.decode(state)
+        decoded.version = v
         return decoded
       }
       case 1:
       case 2:
       case 3: {
         const decoded = encoding4.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -377,12 +376,11 @@ const encoding14 = {
     if (m.trace) encoding14_4.encode(state, m.trace)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = encoding9.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       node: r0,
       checkpoint: (flags & 1) !== 0 ? encoding14_1.decode(state) : null,
       digest: (flags & 2) !== 0 ? encoding14_2.decode(state) : null,
@@ -412,7 +410,6 @@ const encoding15 = {
     encoding1.encode(state, m.views)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding15_1.decode(state)
     const r2 = encoding1.decode(state)
@@ -420,7 +417,7 @@ const encoding15 = {
     const r4 = encoding1.decode(state)
 
     return {
-      version: v,
+      version: 0,
       members: r0,
       pendingIndexers: r1,
       indexers: r2,
@@ -460,7 +457,6 @@ const encoding16 = {
     if (m.entropy) c.fixed32.encode(state, m.entropy)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding16_1.decode(state)
     const r2 = encoding1.decode(state)
@@ -470,7 +466,7 @@ const encoding16 = {
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       members: r0,
       pendingIndexers: r1,
       indexers: r2,
@@ -514,7 +510,6 @@ const encoding18 = {
     if (m.hash) c.fixed8.encode(state, m.hash)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = c.uint.decode(state)
     const r2 = encoding22.decode(state)
@@ -522,7 +517,7 @@ const encoding18 = {
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       timestamp: r0,
       flushes: r1,
       view: r2,
@@ -573,22 +568,23 @@ const encoding19 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0:
       case 1: {
         const decoded = encoding15.decode(state)
+        decoded.version = v
         const map = external0.infoLegacyMap
         return map(decoded)
       }
       case 2: {
         const decoded = encoding16.decode(state)
+        decoded.version = v
         return decoded
       }
       case 3: {
         const decoded = encoding18.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -625,11 +621,10 @@ const encoding20 = {
     if (m.timestamp) c.int.encode(state, m.timestamp)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       isRemoved: (flags & 1) !== 0,
       isOplog: (flags & 2) !== 0,
       weight: c.uint.decode(state),
@@ -686,24 +681,25 @@ const encoding21 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0:
       case 1:
       case 2:
       case 3: {
         const decoded = encoding17.decode(state)
+        decoded.version = v
         const map = external0.memberLegacyMap
         return map(decoded)
       }
       case 4: {
         const decoded = encoding20.decode(state)
+        decoded.version = v
         return decoded
       }
       case 5: {
         const decoded = encoding34.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -953,13 +949,12 @@ const encoding29 = {
     if (m.trusted) encoding29_8.encode(state, m.trusted)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding29_1.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       timestamp: r0,
       links: r1,
       batch: (flags & 1) !== 0 ? encoding24.decode(state) : null,
@@ -1022,29 +1017,32 @@ const encoding30 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0: {
         const decoded = encoding12.decode(state)
+        decoded.version = v
         return decoded
       }
       case 1: {
         const decoded = encoding13.decode(state)
+        decoded.version = v
         return decoded
       }
       case 2: {
         const decoded = encoding14.decode(state)
+        decoded.version = v
         return decoded
       }
       case 3: {
         const decoded = encoding29.decode(state)
+        decoded.version = v
         const map = external0.oplogLegacyMap
         return map(decoded)
       }
       case 4: {
         const decoded = encoding37.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -1154,11 +1152,10 @@ const encoding34 = {
     c.uint.encode(state, m.timestamp)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       isRemoved: (flags & 1) !== 0,
       isOplog: (flags & 2) !== 0,
       isGenesis: (flags & 4) !== 0,
@@ -1226,7 +1223,9 @@ const encoding37 = {
       (m.approvals ? 16 : 0) |
       (m.optimistic ? 32 : 0) |
       (m.value ? 64 : 0) |
-      (m.hash ? 128 : 0)
+      (m.hash ? 128 : 0) |
+      (m.weightHint ? 256 : 0) |
+      (m.drift ? 512 : 0)
 
     c.uint.preencode(state, m.timestamp)
     encoding37_1.preencode(state, m.links)
@@ -1239,6 +1238,8 @@ const encoding37 = {
     if (m.approvals) encoding37_6.preencode(state, m.approvals)
     if (m.value) c.buffer.preencode(state, m.value)
     if (m.hash) c.fixed8.preencode(state, m.hash)
+    if (m.weightHint) c.uint.preencode(state, m.weightHint)
+    if (m.drift) c.uint.preencode(state, m.drift)
   },
   encode(state, m) {
     const flags =
@@ -1249,7 +1250,9 @@ const encoding37 = {
       (m.approvals ? 16 : 0) |
       (m.optimistic ? 32 : 0) |
       (m.value ? 64 : 0) |
-      (m.hash ? 128 : 0)
+      (m.hash ? 128 : 0) |
+      (m.weightHint ? 256 : 0) |
+      (m.drift ? 512 : 0)
 
     c.uint.encode(state, m.timestamp)
     encoding37_1.encode(state, m.links)
@@ -1262,15 +1265,16 @@ const encoding37 = {
     if (m.approvals) encoding37_6.encode(state, m.approvals)
     if (m.value) c.buffer.encode(state, m.value)
     if (m.hash) c.fixed8.encode(state, m.hash)
+    if (m.weightHint) c.uint.encode(state, m.weightHint)
+    if (m.drift) c.uint.encode(state, m.drift)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding37_1.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       timestamp: r0,
       links: r1,
       batch: (flags & 1) !== 0 ? encoding24.decode(state) : null,
@@ -1280,7 +1284,9 @@ const encoding37 = {
       approvals: (flags & 16) !== 0 ? encoding37_6.decode(state) : null,
       optimistic: (flags & 32) !== 0,
       value: (flags & 64) !== 0 ? c.buffer.decode(state) : null,
-      hash: (flags & 128) !== 0 ? c.fixed8.decode(state) : null
+      hash: (flags & 128) !== 0 ? c.fixed8.decode(state) : null,
+      weightHint: (flags & 256) !== 0 ? c.uint.decode(state) : 0,
+      drift: (flags & 512) !== 0 ? c.uint.decode(state) : 0
     }
   }
 }

package/encoding/spec/autobee/schema.json

@@ -93,6 +93,7 @@
     {
       "name": "boot-record",
       "namespace": "autobase-compat",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/boot-record-v0",
@@ -445,6 +446,7 @@
     {
       "name": "system-info",
       "namespace": "autobee",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/info-v1",
@@ -528,6 +530,7 @@
     {
       "name": "system-writer",
       "namespace": "autobee",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/member",
@@ -778,6 +781,7 @@
     {
       "name": "oplog",
       "namespace": "autobee",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/oplog-message-v0",
@@ -1028,6 +1032,16 @@
           "name": "hash",
           "type": "fixed8",
           "version": 1
+        },
+        {
+          "name": "weightHint",
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "drift",
+          "type": "uint",
+          "version": 1
         }
       ]
     }

package/index.js

@@ -5,6 +5,7 @@ const safetyCatch = require('safety-catch')
 const Hyperbee = require('hyperbee2')
 const ID = require('hypercore-id-encoding')
 const rrp = require('resolve-reject-promise')
+const Signal = require('signal-promise')
 const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('autobee-encryption')
 const AutobeeWakeup = require('autobee-wakeup')
 const Hypercore = require('hypercore')
@@ -12,6 +13,7 @@ const crypto = require('hypercore-crypto')
 const c = require('compact-encoding')
 const asserts = require('./lib/asserts.js')
 const boot = require('./lib/boot.js')
+const { DEFAULT_MANIFEST_VERSION } = require('./lib/constants.js')
 const { resolveWeight, currentWeight } = require('./lib/witness.js')
 const encoding = require('./lib/encoding.js')
 const FastForward = require('./lib/fast-forward.js')
@@ -26,6 +28,7 @@ const migrations = require('./lib/migrations.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 const DEFAULT_ACK_THRESHOLD = 32
+const BOOT_NETWORK_TIMEOUT = 5000
 const INTERRUPT = new Error('Apply interrupted')
 
 module.exports = class Autobee extends ReadyResource {
@@ -116,6 +119,9 @@ module.exports = class Autobee extends ReadyResource {
     this.fastForwardTo = null
     this._ffCandidates = new Map()
     this._ffSearching = null
+    this._bumpSignal = new Signal()
+    this._networkBooted = false
+    this._networkBooting = false
 
     this._workingBee = bee
     this._workingView = new ApplyView(this._workingBee, this)
@@ -129,7 +135,6 @@ module.exports = class Autobee extends ReadyResource {
     this._localViewStart = 0
     this._localViewLength = 0
 
-    this._appending = []
     this._draining = null
     this._updating = null
 
@@ -144,6 +149,7 @@ module.exports = class Autobee extends ReadyResource {
     this._now = handlers.now || Date.now // overridable for clock-drift tests
     this._preapply = handlers.preapply || null
     this._preApplied = false
+    this._reindexed = false
     this._warmup = handlers.warmup || null
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
@@ -181,6 +187,10 @@ module.exports = class Autobee extends ReadyResource {
     return this.writers ? this.writers.localWriter.isIndexer : false
   }
 
+  get appending() {
+    return this.writers ? this.writers.localWriter.appending : false
+  }
+
   get writable() {
     return this.writers ? this.writers.writable : false
   }
@@ -202,7 +212,8 @@ module.exports = class Autobee extends ReadyResource {
     await 1
     const result = await this._prebooting
     return {
-      name: 'autobee/' + result.local.id + '/' + name,
+      name: 'autobee/' + result.local.id + '/view/' + name,
+      manifestVersion: DEFAULT_MANIFEST_VERSION,
       encryption: name === 'system' ? this.getSystemEncryption() : this.getViewEncryption(),
       inflightRange: [256, 512]
     }
@@ -377,6 +388,7 @@ module.exports = class Autobee extends ReadyResource {
 
   async _close() {
     this._interrupting = true
+    this._bumpSignal.notify()
 
     // the local core holds the exclusive lock but the local writer closes it
     // early in the teardown, so hand the lock to a detached session that
@@ -580,6 +592,11 @@ module.exports = class Autobee extends ReadyResource {
     this._localFlushes = this.system.flushes
   }
 
+  async _isReindexing() {
+    if (this._networkBooted || !this._ffEnabled || this._migrating) return false
+    return !(await this._isReindexed())
+  }
+
   async _bootOnline() {
     if (!this._bootOnlineGuard.enter()) return
 
@@ -604,6 +621,7 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   bumpSoon() {
+    this._bumpSignal.notify()
     this._bump(false).catch(safetyCatch)
   }
 
@@ -665,6 +683,58 @@ module.exports = class Autobee extends ReadyResource {
     this.emit('error', err)
   }
 
+  async compactMaybe() {
+    if (!this.writers.writable) return 0
+
+    if (await this._isReindexed()) return 0
+
+    const view = await this._reindexBee(this._workingBee)
+    if (view > 0) this.bee.move(this._workingBee.head())
+
+    const system = await this._reindexBee(this.system.bee)
+
+    if (view + system > 0) this._reindexed = true
+
+    return view + system
+  }
+
+  async _isReindexed() {
+    const head = this.system.bee.head()
+    if (head === null) return true
+    if (await this._shouldReindex(head.key)) return false
+
+    const view = this.system.view
+    if (!view || !view.key) return true
+    return !(await this._shouldReindex(view.key))
+  }
+
+  async _reindexBee(bee) {
+    const head = bee.head()
+    const local = bee.context.local
+    if (head === null) return 0
+    if ((await this._shouldReindex(local.key)) && local.length > 0) return 0
+    if (!(await this._shouldReindex(head.key))) return 0
+
+    return bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
+  }
+
+  async _shouldReindex(key, { unknown = false, length = 0, timeout = 0 } = {}) {
+    const core = this.store.get({ key, active: false })
+
+    try {
+      await core.ready()
+
+      if (core.manifest === null && length > 0) {
+        await core.get(length - 1, { raw: true, timeout }).catch(safetyCatch)
+      }
+
+      if (core.manifest === null) return unknown
+      return core.manifest.version > 1 && core.manifest.version < DEFAULT_MANIFEST_VERSION
+    } finally {
+      await core.close()
+    }
+  }
+
   // one-shot user gate: nothing applies until the host has resolved whatever
   // state apply depends on (e.g. legacy views recorded by a migration)
   async _runPreApply() {
@@ -685,6 +755,7 @@ module.exports = class Autobee extends ReadyResource {
       const { head = null, legacy = null } = this.bootFrom
 
       this.bootFrom = null
+      this._networkBooted = true
 
       if (legacy) {
         await this._bootFromSystem(legacy)
@@ -692,8 +763,14 @@ module.exports = class Autobee extends ReadyResource {
         this._wakeup.hint({ key: head.key, length: head.length || 0 })
         await this._bootFromHead(head)
       }
+    } else if (await this._isReindexing()) {
+      this._networkBooted = true
+      await this._bootFromNetwork()
     }
 
+    // preferably get a peer's compacted view during bootFrom
+    if (this.fastForwardTo === null) await this.compactMaybe()
+
     const changes = this._hasUpdate ? new UpdateChanges(this) : null
     if (changes) changes.track()
 
@@ -862,6 +939,7 @@ module.exports = class Autobee extends ReadyResource {
         const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
 
         const ff = await FastForward.fromHeads(this, heads, {
+          skipGap: this._networkBooting,
           timeout: FastForward.DEFAULT_TIMEOUT
         })
 
@@ -1205,17 +1283,21 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _appendAck() {
-    if (!this._acking) return false
+    if (!this._acking && !this._reindexed) return false
     if (!this.writers.writable) return false
 
     if (this.writers.localWriter.pending !== null) return false
-    if ((await this._flushesBehind()) < this._ackThreshold) return false
-    if (await this._allHeadsTrusted()) return false
+
+    if (!this._reindexed) {
+      if ((await this._flushesBehind()) < this._ackThreshold) return false
+      if (await this._allHeadsTrusted()) return false
+    }
 
     const links = this.system.getLinks(this.local.key)
-    const t = Math.max(this._now(), this.system.timestamp)
+    const now = this._now()
+    const t = Math.max(now, this.system.timestamp)
 
-    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null)
+    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null, null, null, t - now)
     return true
   }
 
@@ -1367,6 +1449,14 @@ module.exports = class Autobee extends ReadyResource {
 
     const changed = await this.system.flush(batch, this._workingBee)
 
+    if (this._reindexed && (await this._isReindexed())) {
+      this._reindexed = false
+      await this.local.setUserData(
+        'autobee/head',
+        encoding.encodeBootRecord(this.system.bootRecord())
+      )
+    }
+
     if (this.system.promotions.changed) this._prefetchApprovals().catch(safetyCatch)
 
     if (local) {
@@ -1441,8 +1531,10 @@ module.exports = class Autobee extends ReadyResource {
     const links = this.system.getLinks(this.local.key)
     const hash = this.system.hash
 
-    // never stamp before anything we link
-    const t = Math.max(this._now(), this.system.timestamp)
+    // never stamp before anything we link, but record how far our clock lagged
+    const now = this._now()
+    const t = Math.max(now, this.system.timestamp)
+    const drift = t - now
     const batch = []
 
     const rec = await this.system.get(this.local.key)
@@ -1468,7 +1560,8 @@ module.exports = class Autobee extends ReadyResource {
         optimistic,
         i === 0 ? witness : null,
         i === 0 ? approvals : null,
-        i === 0 ? hash : null
+        i === 0 ? hash : null,
+        drift
       )
       batch.push(node)
     }
@@ -1544,6 +1637,41 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
+  async _bootFromNetwork() {
+    this._networkBooting = true
+
+    try {
+      await this._waitForNetworkBoot()
+    } finally {
+      this._networkBooting = false
+    }
+  }
+
+  async _waitForNetworkBoot() {
+    const deadline = Date.now() + BOOT_NETWORK_TIMEOUT
+
+    this._requestWakeup()
+
+    while (!this._interrupting && !this.closing && this.fastForwardTo === null) {
+      const hints = await this._applyWakeupHints()
+      if (hints.length) this._queueFastForward(hints)
+
+      const remaining = deadline - Date.now()
+      if (remaining <= 0 || this.fastForwardTo !== null) return
+
+      if (this._ffSearching !== null) {
+        await Promise.race([this._ffSearching, this._bumpSignal.wait(remaining)])
+        this._bumpSignal.notify()
+        if (this._ffSearching === null) return
+        continue
+      }
+
+      if (hints.length) return
+
+      await this._bumpSignal.wait(remaining)
+    }
+  }
+
   async _runFastForward(ff, { force = false } = {}) {
     if (this.fastForwardTo !== null || this.fastForwarding !== null) {
       await ff.close()
@@ -1560,7 +1688,8 @@ module.exports = class Autobee extends ReadyResource {
 
     if (!result) return false
 
-    if (!force && flushes - this.system.flushes < FastForward.MIN_GAP) return false
+    // apply ran while we searched, only drop it if it no longer moves us forward
+    if (!force && flushes <= this.system.flushes) return false
 
     this.fastForwardTo = result
     this.ff = rrp()

package/lib/boot.js

@@ -93,16 +93,16 @@ module.exports = async function boot(
     await result.bootstrap.setUserData('referrer', result.key)
     await result.bootstrap.setUserData('autobee/local', result.local.key)
     await result.local.setUserData('referrer', result.key)
+  }
 
-    result.migration = await migrateWithFallback(corestore, result.local, result.bootstrap)
+  result.migration = await migrateWithFallback(corestore, result.local, result.bootstrap)
 
-    if (result.migration) {
-      await result.local.setUserData(
-        'autobee/head',
-        encoding.encodeBootRecord(result.migration.system)
-      )
-      await result.local.setUserData('autobee/encryption', result.migration.encryptionKey)
-    }
+  if (result.migration) {
+    await result.local.setUserData(
+      'autobee/head',
+      encoding.encodeBootRecord(result.migration.system)
+    )
+    await result.local.setUserData('autobee/encryption', result.migration.encryptionKey)
   }
 
   const [systemHead, encryptionKeyBuffer, prevDrainBuffer] = await Promise.all([

package/lib/constants.js

@@ -1,10 +1,12 @@
 const LEGACY_OPLOG_VERSION = 2
 const OPLOG_VERSION = 4
+const DEFAULT_MANIFEST_VERSION = 3
 const LEGACY_AUTOBASE_VERSION = 2
 const AUTOBEE_VERSION = 3
 const DEFAULT_OP_TIMEOUT = 5000
 
 module.exports = {
+  DEFAULT_MANIFEST_VERSION,
   LEGACY_OPLOG_VERSION,
   OPLOG_VERSION,
   LEGACY_AUTOBASE_VERSION,

package/lib/encoding.js

@@ -165,7 +165,9 @@ function decodeOplog(buf) {
       witness: null,
       approvals: null,
       optimistic: !!m.optimistic,
-      value: m.node.value
+      value: m.node.value,
+      weightHint: 0,
+      drift: 0
     }
   }
 

package/lib/fast-forward.js

@@ -7,7 +7,7 @@ const migrations = require('./migrations.js')
 const { LEGACY_AUTOBASE_VERSION, LEGACY_OPLOG_VERSION } = require('./constants.js')
 
 const DEFAULT_OP_TIMEOUT = 5_000
-const MIN_FF_GAP = 32
+const MIN_FF_GAP = 16
 
 class FastForward {
   constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT, conservative = false } = {}) {
@@ -38,9 +38,8 @@ class FastForward {
   }
 
   static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
-  static MIN_GAP = MIN_FF_GAP
 
-  static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
+  static async fromHead(auto, head, trusted, { force = false, skipGap = false, timeout = 0 } = {}) {
     const conservative = !force && auto._conservativeFF
 
     const oplog = await FastForward.flushHead(auto, head, { timeout })
@@ -52,7 +51,7 @@ class FastForward {
     // legacy nodes from non-indexers have no system info to fast-forward from
     if (!oplog.op.views || !verified.op.views) return null
 
-    if (!force && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
+    if (!force && !skipGap && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
       return null
     }
 
@@ -62,7 +61,7 @@ class FastForward {
     })
   }
 
-  static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
+  static async fromHeads(auto, heads, { force = false, skipGap = false, timeout = 0 } = {}) {
     const reference = auto._workingView.view
 
     const promises = []
@@ -72,14 +71,12 @@ class FastForward {
       promises.push(FastForward.flushHead(auto, head, { timeout }))
     }
 
-    const ops = await Promise.all(promises)
+    const ops = await FastForward.reindexedCandidates(auto, await Promise.all(promises), {
+      timeout: timeout || DEFAULT_OP_TIMEOUT
+    })
     if (auto.fastForwarding || auto.fastForwardTo) return null
 
-    const trust = await Promise.all(
-      ops.map((res) => {
-        return res === null ? false : auto.trusted.isTrusted(res.key, reference)
-      })
-    )
+    const trust = await Promise.all(ops.map((res) => auto.trusted.isTrusted(res.key, reference)))
     if (auto.fastForwarding || auto.fastForwardTo) return null
 
     const candidates = []
@@ -89,8 +86,7 @@ class FastForward {
 
     for (let i = 0; i < ops.length; i++) {
       const res = ops[i]
-      if (res === null || !res.op.views) continue
-      if (!force && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
+      if (!force && !skipGap && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
 
       if (!trust[i]) {
         candidates.push(res)
@@ -104,7 +100,7 @@ class FastForward {
     }
 
     if (bestTrusted !== null) {
-      return FastForward.fromHead(auto, bestTrusted, null, { force, timeout })
+      return FastForward.fromHead(auto, bestTrusted, null, { force, skipGap, timeout })
     }
 
     candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
@@ -114,13 +110,38 @@ class FastForward {
       if (trusted === null) continue
       if (auto.fastForwarding || auto.fastForwardTo) return null
 
-      const ff = await FastForward.fromHead(auto, res, trusted, { force, timeout })
+      const ff = await FastForward.fromHead(auto, res, trusted, { force, skipGap, timeout })
       if (ff !== null) return ff
     }
 
     return null
   }
 
+  static async reindexedCandidates(auto, ops, { timeout }) {
+    const withViews = ops.filter((res) => res !== null && res.op.views)
+    const reindexed = await Promise.all(
+      withViews.map((res) => FastForward.isReindexed(auto, res, timeout))
+    )
+    return withViews.filter((res, i) => reindexed[i])
+  }
+
+  static async isReindexed(auto, res, timeout) {
+    const { system, view } = res.op.views
+    const checks = [system, view]
+      .filter((v) => !!v)
+      .map((range) => {
+        const { key, length } = batchToHead(range)
+        if (length === 0) return false
+        return auto._shouldReindex(key, { unknown: true, length, timeout })
+      })
+
+    for (const shouldReindex of await Promise.all(checks)) {
+      if (shouldReindex) return false
+    }
+
+    return true
+  }
+
   static async flushHead(auto, head, opts) {
     const core = auto.openCore(head.key)
 

package/lib/migrations.js

@@ -173,14 +173,18 @@ function decodeLegacySystemInfo(buffer) {
   const state = { start: 0, end: buffer.length, buffer }
   const version = c.uint.decode(state)
 
-  state.start--
-
   switch (version) {
     case 0:
-    case 1:
-      return SystemInfoV1.decode(state)
-    case 2:
-      return SystemInfoV2.decode(state)
+    case 1: {
+      const decoded = SystemInfoV1.decode(state)
+      decoded.version = version
+      return decoded
+    }
+    case 2: {
+      const decoded = SystemInfoV2.decode(state)
+      decoded.version = version
+      return decoded
+    }
     default:
       bail('Expected legacy system info')
   }

package/lib/writers.js

@@ -92,7 +92,7 @@ class ActiveWriters {
     return this.localWriter.getLatest()
   }
 
-  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
+  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals, hash, drift) {
     return this.localWriter.append(
       value,
       timestamp,
@@ -101,7 +101,8 @@ class ActiveWriters {
       optimistic,
       witness,
       approvals,
-      hash
+      hash,
+      drift
     )
   }
 
@@ -725,7 +726,7 @@ class Writer {
     return this.pending !== null
   }
 
-  append(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
+  append(value, timestamp, batch, links, optimistic, witness, approvals, hash, drift) {
     if (this.pending === null) this.pending = []
 
     const oplog = {
@@ -739,7 +740,9 @@ class Writer {
       approvals,
       optimistic: optimistic && !this.writers.writable,
       value,
-      hash
+      hash,
+      weightHint: 0,
+      drift
     }
 
     const node = createNode(this.core, this.appendLength++, -1, oplog)
@@ -773,7 +776,9 @@ class Writer {
 
     const buffers = []
     for (let i = 0; i < this.processed; i++) {
-      buffers.push(encoding.encodeOplog(this.pending[i]))
+      const node = this.pending[i]
+      node.weightHint = node.weight > 0 ? node.weight : 0
+      buffers.push(encoding.encodeOplog(node))
     }
 
     try {
@@ -817,7 +822,9 @@ function createNode(core, length, weight, oplog) {
     approvals: oplog.approvals || null,
     optimistic: oplog.optimistic,
     value: oplog.value,
-    hash: oplog.hash || null
+    hash: oplog.hash || null,
+    weightHint: oplog.weightHint || 0,
+    drift: oplog.drift || 0
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.10.0",
+  "version": "2.11.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -27,15 +27,16 @@
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
     "compact-encoding": "^3.3.0",
-    "hyperbee2": "^2.16.0",
-    "hypercore": "^11.33.2",
+    "hyperbee2": "^2.17.0",
+    "hypercore": "^11.37.0",
     "hypercore-id-encoding": "^1.3.0",
-    "hyperschema": "^1.20.1",
+    "hyperschema": "^1.26.2",
     "read-write-mutexify": "^2.1.0",
     "ready-guard": "^1.0.0",
     "ready-resource": "^1.2.0",
     "resolve-reject-promise": "^1.1.0",
     "safety-catch": "^1.0.3",
+    "signal-promise": "^1.0.3",
     "sodium-native": "^5.1.0"
   },
   "devDependencies": {