autobee 2.9.6 → 2.10.1

package/encoding/legacy.js

@@ -215,7 +215,6 @@ const OplogMessageV1 = {
     throw new Error('Encoding not supported')
   },
   decode(state) {
-    c.uint.decode(state) // version
     const maxSupportedVersion = c.uint.decode(state)
 
     const flags = c.uint.decode(state)
@@ -246,7 +245,6 @@ const OplogMessageV0 = {
     throw new Error('Encoding not supported')
   },
   decode(state) {
-    c.uint.decode(state) // version
     const flags = c.uint.decode(state)
 
     const isCheckpointer = (flags & 1) !== 0
@@ -276,6 +274,7 @@ const SystemWriterV0 = {
     throw new Error('Encoding not supported')
   },
   decode(state) {
+    state.start-- // Undo the hack to nest this in a versioned struct w/o a version
     const flags = c.uint.decode(state)
 
     return {
@@ -328,7 +327,9 @@ function oplogLegacyMap(m) {
     witness: null,
     approvals: null,
     optimistic: m.optimistic,
-    value: m.value
+    value: m.value,
+    weightHint: 0,
+    drift: 0
   }
 }
 

package/encoding/spec/autobee/index.js

@@ -108,13 +108,12 @@ const encoding4 = {
     if (m.recoveries) c.uint.encode(state, m.recoveries)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.fixed32.decode(state)
     const r1 = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       key: r0,
       systemLength: r1,
       indexersUpdated: (flags & 1) !== 0,
@@ -159,18 +158,18 @@ const encoding5 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0: {
         const decoded = encoding3.decode(state)
+        decoded.version = v
         return decoded
       }
       case 1:
       case 2:
       case 3: {
         const decoded = encoding4.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -377,12 +376,11 @@ const encoding14 = {
     if (m.trace) encoding14_4.encode(state, m.trace)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = encoding9.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       node: r0,
       checkpoint: (flags & 1) !== 0 ? encoding14_1.decode(state) : null,
       digest: (flags & 2) !== 0 ? encoding14_2.decode(state) : null,
@@ -412,7 +410,6 @@ const encoding15 = {
     encoding1.encode(state, m.views)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding15_1.decode(state)
     const r2 = encoding1.decode(state)
@@ -420,7 +417,7 @@ const encoding15 = {
     const r4 = encoding1.decode(state)
 
     return {
-      version: v,
+      version: 0,
       members: r0,
       pendingIndexers: r1,
       indexers: r2,
@@ -460,7 +457,6 @@ const encoding16 = {
     if (m.entropy) c.fixed32.encode(state, m.entropy)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding16_1.decode(state)
     const r2 = encoding1.decode(state)
@@ -470,7 +466,7 @@ const encoding16 = {
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       members: r0,
       pendingIndexers: r1,
       indexers: r2,
@@ -514,7 +510,6 @@ const encoding18 = {
     if (m.hash) c.fixed8.encode(state, m.hash)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = c.uint.decode(state)
     const r2 = encoding22.decode(state)
@@ -522,7 +517,7 @@ const encoding18 = {
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       timestamp: r0,
       flushes: r1,
       view: r2,
@@ -573,22 +568,23 @@ const encoding19 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0:
       case 1: {
         const decoded = encoding15.decode(state)
+        decoded.version = v
         const map = external0.infoLegacyMap
         return map(decoded)
       }
       case 2: {
         const decoded = encoding16.decode(state)
+        decoded.version = v
         return decoded
       }
       case 3: {
         const decoded = encoding18.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -625,11 +621,10 @@ const encoding20 = {
     if (m.timestamp) c.int.encode(state, m.timestamp)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       isRemoved: (flags & 1) !== 0,
       isOplog: (flags & 2) !== 0,
       weight: c.uint.decode(state),
@@ -686,24 +681,25 @@ const encoding21 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0:
       case 1:
       case 2:
       case 3: {
         const decoded = encoding17.decode(state)
+        decoded.version = v
         const map = external0.memberLegacyMap
         return map(decoded)
       }
       case 4: {
         const decoded = encoding20.decode(state)
+        decoded.version = v
         return decoded
       }
       case 5: {
         const decoded = encoding34.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -953,13 +949,12 @@ const encoding29 = {
     if (m.trusted) encoding29_8.encode(state, m.trusted)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding29_1.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       timestamp: r0,
       links: r1,
       batch: (flags & 1) !== 0 ? encoding24.decode(state) : null,
@@ -1022,29 +1017,32 @@ const encoding30 = {
     }
   },
   decode(state) {
-    const start = state.start
     const v = c.uint.decode(state)
-    state.start = start
     switch (v) {
       case 0: {
         const decoded = encoding12.decode(state)
+        decoded.version = v
         return decoded
       }
       case 1: {
         const decoded = encoding13.decode(state)
+        decoded.version = v
         return decoded
       }
       case 2: {
         const decoded = encoding14.decode(state)
+        decoded.version = v
         return decoded
       }
       case 3: {
         const decoded = encoding29.decode(state)
+        decoded.version = v
         const map = external0.oplogLegacyMap
         return map(decoded)
       }
       case 4: {
         const decoded = encoding37.decode(state)
+        decoded.version = v
         return decoded
       }
       default:
@@ -1154,11 +1152,10 @@ const encoding34 = {
     c.uint.encode(state, m.timestamp)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       isRemoved: (flags & 1) !== 0,
       isOplog: (flags & 2) !== 0,
       isGenesis: (flags & 4) !== 0,
@@ -1226,7 +1223,9 @@ const encoding37 = {
       (m.approvals ? 16 : 0) |
       (m.optimistic ? 32 : 0) |
       (m.value ? 64 : 0) |
-      (m.hash ? 128 : 0)
+      (m.hash ? 128 : 0) |
+      (m.weightHint ? 256 : 0) |
+      (m.drift ? 512 : 0)
 
     c.uint.preencode(state, m.timestamp)
     encoding37_1.preencode(state, m.links)
@@ -1239,6 +1238,8 @@ const encoding37 = {
     if (m.approvals) encoding37_6.preencode(state, m.approvals)
     if (m.value) c.buffer.preencode(state, m.value)
     if (m.hash) c.fixed8.preencode(state, m.hash)
+    if (m.weightHint) c.uint.preencode(state, m.weightHint)
+    if (m.drift) c.uint.preencode(state, m.drift)
   },
   encode(state, m) {
     const flags =
@@ -1249,7 +1250,9 @@ const encoding37 = {
       (m.approvals ? 16 : 0) |
       (m.optimistic ? 32 : 0) |
       (m.value ? 64 : 0) |
-      (m.hash ? 128 : 0)
+      (m.hash ? 128 : 0) |
+      (m.weightHint ? 256 : 0) |
+      (m.drift ? 512 : 0)
 
     c.uint.encode(state, m.timestamp)
     encoding37_1.encode(state, m.links)
@@ -1262,15 +1265,16 @@ const encoding37 = {
     if (m.approvals) encoding37_6.encode(state, m.approvals)
     if (m.value) c.buffer.encode(state, m.value)
     if (m.hash) c.fixed8.encode(state, m.hash)
+    if (m.weightHint) c.uint.encode(state, m.weightHint)
+    if (m.drift) c.uint.encode(state, m.drift)
   },
   decode(state) {
-    const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
     const r1 = encoding37_1.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: v,
+      version: 0,
       timestamp: r0,
       links: r1,
       batch: (flags & 1) !== 0 ? encoding24.decode(state) : null,
@@ -1280,7 +1284,9 @@ const encoding37 = {
       approvals: (flags & 16) !== 0 ? encoding37_6.decode(state) : null,
       optimistic: (flags & 32) !== 0,
       value: (flags & 64) !== 0 ? c.buffer.decode(state) : null,
-      hash: (flags & 128) !== 0 ? c.fixed8.decode(state) : null
+      hash: (flags & 128) !== 0 ? c.fixed8.decode(state) : null,
+      weightHint: (flags & 256) !== 0 ? c.uint.decode(state) : 0,
+      drift: (flags & 512) !== 0 ? c.uint.decode(state) : 0
     }
   }
 }

package/encoding/spec/autobee/schema.json

@@ -93,6 +93,7 @@
     {
       "name": "boot-record",
       "namespace": "autobase-compat",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/boot-record-v0",
@@ -445,6 +446,7 @@
     {
       "name": "system-info",
       "namespace": "autobee",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/info-v1",
@@ -528,6 +530,7 @@
     {
       "name": "system-writer",
       "namespace": "autobee",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/member",
@@ -778,6 +781,7 @@
     {
       "name": "oplog",
       "namespace": "autobee",
+      "framed": false,
       "versions": [
         {
           "type": "@autobase-compat/oplog-message-v0",
@@ -1028,6 +1032,16 @@
           "name": "hash",
           "type": "fixed8",
           "version": 1
+        },
+        {
+          "name": "weightHint",
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "drift",
+          "type": "uint",
+          "version": 1
         }
       ]
     }

package/index.js

@@ -114,6 +114,8 @@ module.exports = class Autobee extends ReadyResource {
     this.ff = null
     this.fastForwarding = null
     this.fastForwardTo = null
+    this._ffCandidates = new Map()
+    this._ffSearching = null
 
     this._workingBee = bee
     this._workingView = new ApplyView(this._workingBee, this)
@@ -127,7 +129,6 @@ module.exports = class Autobee extends ReadyResource {
     this._localViewStart = 0
     this._localViewLength = 0
 
-    this._appending = []
     this._draining = null
     this._updating = null
 
@@ -179,6 +180,10 @@ module.exports = class Autobee extends ReadyResource {
     return this.writers ? this.writers.localWriter.isIndexer : false
   }
 
+  get appending() {
+    return this.writers ? this.writers.localWriter.appending : false
+  }
+
   get writable() {
     return this.writers ? this.writers.writable : false
   }
@@ -425,6 +430,7 @@ module.exports = class Autobee extends ReadyResource {
     // rugpull the rest
     await this.store.close()
 
+    if (this._ffSearching) await this._ffSearching
     if (this._updating) await this._updating
     if (this._draining) await this._draining
 
@@ -816,8 +822,22 @@ module.exports = class Autobee extends ReadyResource {
     const hints = await this._applyWakeupHints()
     if (!hints.length) return
 
+    this._queueFastForward(hints)
+  }
+
+  _queueFastForward(hints) {
     if (!this._ffEnabled) return
 
+    for (const { key, length } of hints) {
+      if (length === 0) continue
+
+      const hex = b4a.toString(key, 'hex')
+      const previous = this._ffCandidates.get(hex)
+      if (previous === undefined || previous < length) this._ffCandidates.set(hex, length)
+    }
+
+    if (!this._ffCandidates.size || this._ffSearching !== null) return
+
     // a scheduled fast-forward is applied by the drain before we look again
     if (this.fastForwardTo !== null || this.fastForwarding !== null) return
     if (this._interrupting || this.closing || this.bootFrom) return
@@ -829,14 +849,34 @@ module.exports = class Autobee extends ReadyResource {
       return
     }
 
+    this._ffSearching = this._searchFastForward().finally(() => {
+      this._ffSearching = null
+    })
+  }
+
+  async _searchFastForward() {
     try {
-      const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
+      while (this._ffCandidates.size) {
+        if (this.fastForwardTo !== null || this.fastForwarding !== null) return
+        if (this._interrupting || this.closing || this.bootFrom) return
 
-      const ff = await FastForward.fromHeads(this, heads, {
-        timeout: FastForward.DEFAULT_TIMEOUT
-      })
+        const hints = flushCandidates(this._ffCandidates)
 
-      if (ff !== null) await this._runFastForward(ff)
+        const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
+
+        const ff = await FastForward.fromHeads(this, heads, {
+          timeout: FastForward.DEFAULT_TIMEOUT
+        })
+
+        if (ff === null) continue
+
+        if (this._interrupting || this.closing) {
+          await ff.close()
+          return
+        }
+
+        if (await this._runFastForward(ff)) return
+      }
     } catch (err) {
       safetyCatch(err)
     }
@@ -1176,9 +1216,10 @@ module.exports = class Autobee extends ReadyResource {
     if (await this._allHeadsTrusted()) return false
 
     const links = this.system.getLinks(this.local.key)
-    const t = Math.max(this._now(), this.system.timestamp)
+    const now = this._now()
+    const t = Math.max(now, this.system.timestamp)
 
-    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null)
+    this.writers.appendLocal(null, t, { start: 0, end: 0 }, links, false, null, null, null, t - now)
     return true
   }
 
@@ -1404,8 +1445,10 @@ module.exports = class Autobee extends ReadyResource {
     const links = this.system.getLinks(this.local.key)
     const hash = this.system.hash
 
-    // never stamp before anything we link
-    const t = Math.max(this._now(), this.system.timestamp)
+    // never stamp before anything we link, but record how far our clock lagged
+    const now = this._now()
+    const t = Math.max(now, this.system.timestamp)
+    const drift = t - now
     const batch = []
 
     const rec = await this.system.get(this.local.key)
@@ -1431,7 +1474,8 @@ module.exports = class Autobee extends ReadyResource {
         optimistic,
         i === 0 ? witness : null,
         i === 0 ? approvals : null,
-        i === 0 ? hash : null
+        i === 0 ? hash : null,
+        drift
       )
       batch.push(node)
     }
@@ -1475,7 +1519,7 @@ module.exports = class Autobee extends ReadyResource {
     const ff = await FastForward.fromHead(this, head, null, { force: true, timeout })
     if (ff === null) return null
 
-    if (!(await this._runFastForward(ff))) return null
+    if (!(await this._runFastForward(ff, { force: true }))) return null
 
     return this.ff.promise
   }
@@ -1484,7 +1528,7 @@ module.exports = class Autobee extends ReadyResource {
   async _bootFromSystem(system) {
     try {
       const ff = new FastForward(this, system, { timeout: FastForward.DEFAULT_TIMEOUT })
-      return await this._runFastForward(ff)
+      return await this._runFastForward(ff, { force: true })
     } catch (err) {
       safetyCatch(err)
       return false
@@ -1500,14 +1544,14 @@ module.exports = class Autobee extends ReadyResource {
 
       const ff = await FastForward.fromHead(this, oplog, null, { force: true, timeout })
 
-      return ff !== null && (await this._runFastForward(ff))
+      return ff !== null && (await this._runFastForward(ff, { force: true }))
     } catch (err) {
       safetyCatch(err)
       return false
     }
   }
 
-  async _runFastForward(ff) {
+  async _runFastForward(ff, { force = false } = {}) {
     if (this.fastForwardTo !== null || this.fastForwarding !== null) {
       await ff.close()
       return false
@@ -1516,12 +1560,16 @@ module.exports = class Autobee extends ReadyResource {
     this.fastForwarding = ff
 
     const result = await ff.run()
+    const flushes = ff.system.flushes
     await ff.close()
 
     if (this.fastForwarding === ff) this.fastForwarding = null
 
     if (!result) return false
 
+    // apply ran while we searched, only drop it if it no longer moves us forward
+    if (!force && flushes <= this.system.flushes) return false
+
     this.fastForwardTo = result
     this.ff = rrp()
 
@@ -1561,7 +1609,7 @@ module.exports = class Autobee extends ReadyResource {
     this.fastForwardTo = null
 
     // process any wakeup while fast-forward itself was in flight
-    await this._applyWakeupHints()
+    this._queueFastForward(await this._applyWakeupHints())
     await this.writers.refresh()
 
     // we moved, so ask our peers to tell us their heads again
@@ -1597,6 +1645,18 @@ function getBootOption(boot) {
   return boot
 }
 
+function flushCandidates(candidates) {
+  const hints = []
+
+  for (const [hex, length] of candidates) {
+    hints.push({ key: b4a.from(hex, 'hex'), length })
+  }
+
+  candidates.clear()
+
+  return hints
+}
+
 function noop() {}
 
 function createAnchorCore(store, prologue, manifestData) {

package/lib/boot.js

@@ -93,16 +93,16 @@ module.exports = async function boot(
     await result.bootstrap.setUserData('referrer', result.key)
     await result.bootstrap.setUserData('autobee/local', result.local.key)
     await result.local.setUserData('referrer', result.key)
+  }
 
-    result.migration = await migrateWithFallback(corestore, result.local, result.bootstrap)
+  result.migration = await migrateWithFallback(corestore, result.local, result.bootstrap)
 
-    if (result.migration) {
-      await result.local.setUserData(
-        'autobee/head',
-        encoding.encodeBootRecord(result.migration.system)
-      )
-      await result.local.setUserData('autobee/encryption', result.migration.encryptionKey)
-    }
+  if (result.migration) {
+    await result.local.setUserData(
+      'autobee/head',
+      encoding.encodeBootRecord(result.migration.system)
+    )
+    await result.local.setUserData('autobee/encryption', result.migration.encryptionKey)
   }
 
   const [systemHead, encryptionKeyBuffer, prevDrainBuffer] = await Promise.all([

package/lib/encoding.js

@@ -165,7 +165,9 @@ function decodeOplog(buf) {
       witness: null,
       approvals: null,
       optimistic: !!m.optimistic,
-      value: m.node.value
+      value: m.node.value,
+      weightHint: 0,
+      drift: 0
     }
   }
 

package/lib/migrations.js

@@ -173,14 +173,18 @@ function decodeLegacySystemInfo(buffer) {
   const state = { start: 0, end: buffer.length, buffer }
   const version = c.uint.decode(state)
 
-  state.start--
-
   switch (version) {
     case 0:
-    case 1:
-      return SystemInfoV1.decode(state)
-    case 2:
-      return SystemInfoV2.decode(state)
+    case 1: {
+      const decoded = SystemInfoV1.decode(state)
+      decoded.version = version
+      return decoded
+    }
+    case 2: {
+      const decoded = SystemInfoV2.decode(state)
+      decoded.version = version
+      return decoded
+    }
     default:
       bail('Expected legacy system info')
   }

package/lib/writers.js

@@ -92,7 +92,7 @@ class ActiveWriters {
     return this.localWriter.getLatest()
   }
 
-  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
+  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals, hash, drift) {
     return this.localWriter.append(
       value,
       timestamp,
@@ -101,7 +101,8 @@ class ActiveWriters {
       optimistic,
       witness,
       approvals,
-      hash
+      hash,
+      drift
     )
   }
 
@@ -725,7 +726,7 @@ class Writer {
     return this.pending !== null
   }
 
-  append(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
+  append(value, timestamp, batch, links, optimistic, witness, approvals, hash, drift) {
     if (this.pending === null) this.pending = []
 
     const oplog = {
@@ -739,7 +740,9 @@ class Writer {
       approvals,
       optimistic: optimistic && !this.writers.writable,
       value,
-      hash
+      hash,
+      weightHint: 0,
+      drift
     }
 
     const node = createNode(this.core, this.appendLength++, -1, oplog)
@@ -773,7 +776,9 @@ class Writer {
 
     const buffers = []
     for (let i = 0; i < this.processed; i++) {
-      buffers.push(encoding.encodeOplog(this.pending[i]))
+      const node = this.pending[i]
+      node.weightHint = node.weight > 0 ? node.weight : 0
+      buffers.push(encoding.encodeOplog(node))
     }
 
     try {
@@ -817,7 +822,9 @@ function createNode(core, length, weight, oplog) {
     approvals: oplog.approvals || null,
     optimistic: oplog.optimistic,
     value: oplog.value,
-    hash: oplog.hash || null
+    hash: oplog.hash || null,
+    weightHint: oplog.weightHint || 0,
+    drift: oplog.drift || 0
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.9.6",
+  "version": "2.10.1",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -30,7 +30,7 @@
     "hyperbee2": "^2.16.0",
     "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",
-    "hyperschema": "^1.20.1",
+    "hyperschema": "^1.26.2",
     "read-write-mutexify": "^2.1.0",
     "ready-guard": "^1.0.0",
     "ready-resource": "^1.2.0",