autobee 2.5.5 → 2.7.0

package/README.md

@@ -81,6 +81,7 @@ Options:
   optimistic: true,              // allow optimistic writes from unknown writers
   isTrusted (key, reference) {}, // do we trust this writer, see Fast-forward
   mostRecentTrusted (target, reference) {}, // the head we vouch for, see Fast-forward
+  warmup (view) {},              // prepare a candidate view, see Fast-forward
   ackThreshold: 32,              // flushes we may fall behind before acking, see Acking
   fastForward: {}                // see Fast-forward
 }
@@ -255,6 +256,14 @@ Return the oplog head you most recently vouched for, given the `target` view bei
 
 Called at flush time to stamp your own oplog (with your view as `target` and a `null` reference), and again per candidate during discovery.
 
+#### `warmup(view)`
+
+Prepare a candidate `view` before the fast-forward onto it is applied.
+
+Called as soon as the candidate's view head is known, and runs alongside the block downloads the fast-forward is already doing rather than after them. Throw or reject to reject the candidate: the fast-forward is abandoned and the usual apply path catches up instead.
+
+`view` is opened and closed through the `open` and `close` handlers, so don't build or close a db of your own.
+
 #### `fastForward.boot`
 
 ```js

package/encoding/spec/autobee/index.js

@@ -494,13 +494,14 @@ const encoding18 = {
     c.uint.preencode(state, m.flushes)
     encoding22.preencode(state, m.view)
     encoding18_3.preencode(state, m.heads)
-    state.end++ // max flag is 2 so always one byte
+    state.end++ // max flag is 4 so always one byte
 
     if (m.indexers) encoding18_4.preencode(state, m.indexers)
     if (m.pending) encoding18_5.preencode(state, m.pending)
+    if (m.hash) c.fixed8.preencode(state, m.hash)
   },
   encode(state, m) {
-    const flags = (m.indexers ? 1 : 0) | (m.pending ? 2 : 0)
+    const flags = (m.indexers ? 1 : 0) | (m.pending ? 2 : 0) | (m.hash ? 4 : 0)
 
     c.uint.encode(state, m.timestamp)
     c.uint.encode(state, m.flushes)
@@ -510,6 +511,7 @@ const encoding18 = {
 
     if (m.indexers) encoding18_4.encode(state, m.indexers)
     if (m.pending) encoding18_5.encode(state, m.pending)
+    if (m.hash) c.fixed8.encode(state, m.hash)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -526,7 +528,8 @@ const encoding18 = {
       view: r2,
       heads: r3,
       indexers: (flags & 1) !== 0 ? encoding18_4.decode(state) : null,
-      pending: (flags & 2) !== 0 ? encoding18_5.decode(state) : null
+      pending: (flags & 2) !== 0 ? encoding18_5.decode(state) : null,
+      hash: (flags & 4) !== 0 ? c.fixed8.decode(state) : null
     }
   }
 }
@@ -1215,9 +1218,19 @@ const encoding37_6 = c.array(encoding36)
 // @autobee/oplog-message-v4
 const encoding37 = {
   preencode(state, m) {
+    const flags =
+      (m.batch ? 1 : 0) |
+      (m.views ? 2 : 0) |
+      (m.trusted ? 4 : 0) |
+      (m.witness ? 8 : 0) |
+      (m.approvals ? 16 : 0) |
+      (m.optimistic ? 32 : 0) |
+      (m.value ? 64 : 0) |
+      (m.hash ? 128 : 0)
+
     c.uint.preencode(state, m.timestamp)
     encoding37_1.preencode(state, m.links)
-    state.end++ // max flag is 64 so always one byte
+    c.uint.preencode(state, flags)
 
     if (m.batch) encoding24.preencode(state, m.batch)
     if (m.views) encoding25.preencode(state, m.views)
@@ -1225,6 +1238,7 @@ const encoding37 = {
     if (m.witness) encoding35.preencode(state, m.witness)
     if (m.approvals) encoding37_6.preencode(state, m.approvals)
     if (m.value) c.buffer.preencode(state, m.value)
+    if (m.hash) c.fixed8.preencode(state, m.hash)
   },
   encode(state, m) {
     const flags =
@@ -1234,7 +1248,8 @@ const encoding37 = {
       (m.witness ? 8 : 0) |
       (m.approvals ? 16 : 0) |
       (m.optimistic ? 32 : 0) |
-      (m.value ? 64 : 0)
+      (m.value ? 64 : 0) |
+      (m.hash ? 128 : 0)
 
     c.uint.encode(state, m.timestamp)
     encoding37_1.encode(state, m.links)
@@ -1246,6 +1261,7 @@ const encoding37 = {
     if (m.witness) encoding35.encode(state, m.witness)
     if (m.approvals) encoding37_6.encode(state, m.approvals)
     if (m.value) c.buffer.encode(state, m.value)
+    if (m.hash) c.fixed8.encode(state, m.hash)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -1263,7 +1279,8 @@ const encoding37 = {
       witness: (flags & 8) !== 0 ? encoding35.decode(state) : null,
       approvals: (flags & 16) !== 0 ? encoding37_6.decode(state) : null,
       optimistic: (flags & 32) !== 0,
-      value: (flags & 64) !== 0 ? c.buffer.decode(state) : null
+      value: (flags & 64) !== 0 ? c.buffer.decode(state) : null,
+      hash: (flags & 128) !== 0 ? c.fixed8.decode(state) : null
     }
   }
 }

package/encoding/spec/autobee/schema.json

@@ -434,6 +434,11 @@
           "array": true,
           "type": "bool",
           "version": 1
+        },
+        {
+          "name": "hash",
+          "type": "fixed8",
+          "version": 1
         }
       ]
     },
@@ -1018,6 +1023,11 @@
           "name": "value",
           "type": "buffer",
           "version": 1
+        },
+        {
+          "name": "hash",
+          "type": "fixed8",
+          "version": 1
         }
       ]
     }

package/index.js

@@ -134,6 +134,7 @@ module.exports = class Autobee extends ReadyResource {
     this._now = handlers.now || Date.now // overridable for clock-drift tests
     this._preapply = handlers.preapply || null
     this._preApplied = false
+    this._warmup = handlers.warmup || null
     this._hasApply = !!handlers.apply
     this._hasUpdate = !!handlers.update
     this._needsUpdate = false
@@ -440,7 +441,11 @@ module.exports = class Autobee extends ReadyResource {
     const v = oplog.op.views.view
     if (!v) return null
 
-    return new ApplyView(this.bee.checkout({ key: v.key, length: v.start + v.length }), this)
+    return this.openView({ key: v.key, length: v.start + v.length })
+  }
+
+  openView(head) {
+    return new ApplyView(this.bee.checkout({ key: head.key, length: head.length }), this)
   }
 
   openCore(key) {
@@ -1191,9 +1196,14 @@ module.exports = class Autobee extends ReadyResource {
     node.weight = await resolveWeight(this, node)
     for (const n of batch) n.weight = node.weight
 
-    // if (topo.isLinkingAll(node, this.system.heads)) {
-    //   return { undo: null, view: null, tip: [batch] }
-    // }
+    if (
+      node.hash &&
+      this.system.hash &&
+      b4a.equals(node.hash, this.system.hash) &&
+      topo.isLinkingAll(node, this.system.heads)
+    ) {
+      return { undo: null, view: null, tip: [batch] }
+    }
 
     const t = await topo.sort(this, batch)
 
@@ -1347,6 +1357,7 @@ module.exports = class Autobee extends ReadyResource {
     await this.local.ready()
 
     const links = this.system.getLinks(this.local.key)
+    const hash = this.system.hash
 
     // never stamp before anything we link
     const t = Math.max(this._now(), this.system.timestamp)
@@ -1374,7 +1385,8 @@ module.exports = class Autobee extends ReadyResource {
         lnk,
         optimistic,
         i === 0 ? witness : null,
-        i === 0 ? approvals : null
+        i === 0 ? approvals : null,
+        i === 0 ? hash : null
       )
       batch.push(node)
     }

package/lib/fast-forward.js

@@ -30,6 +30,8 @@ class FastForward {
     this.running = null
     this.failed = false
     this.cores = []
+    this.warmup = null
+    this.warmupView = null
   }
 
   static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
@@ -170,6 +172,8 @@ class FastForward {
 
     const promises = []
 
+    if (this._startWarmup()) promises.push(this.warmup)
+
     // ensure local key is locally available always
     promises.push(this.system.get(this.auto.local.key, { timeout: this.timeout }))
 
@@ -199,6 +203,17 @@ class FastForward {
     }
   }
 
+  _startWarmup() {
+    if (!this.auto._warmup || !this.system.view.length) return false
+
+    this.warmupView = this.auto.openView(this.system.view)
+
+    // ensure thenable
+    this.warmup = Promise.resolve(this.auto._warmup(this.warmupView.view))
+
+    return true
+  }
+
   async _resolveLength() {
     const core = this.auto.store.get({ key: this.head.key })
     this.cores.push(core)
@@ -278,6 +293,22 @@ class FastForward {
   async close() {
     this.destroyed = true
     this.clearRequests()
+
+    // close view to cancel inflight reads
+    if (this.warmupView) {
+      const view = this.warmupView
+      this.warmupView = null
+      await view.close()
+    }
+
+    if (this.warmup) {
+      try {
+        await this.warmup
+      } catch (err) {
+        safetyCatch(err)
+      }
+    }
+
     if (this.system) await this.system.close()
     for (const core of this.cores) await core.close()
   }

package/lib/system.js

@@ -1,5 +1,7 @@
 const Hyperbee = require('hyperbee2')
 const Promotions = require('./promotions.js')
+const sodium = require('sodium-native')
+const c = require('compact-encoding')
 const b4a = require('b4a')
 
 const { getViewMap, getCoreManifest } = require('./migrations.js')
@@ -13,6 +15,10 @@ const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 const EMPTY_HEAD = { length: 0, key: null }
 const EMPTY = b4a.from([])
 
+const HASH = c.fixed(sodium.crypto_shorthash_BYTES)
+const HASH_KEY = b4a.from('autobee-orderkey') // crypto_shorthash_KEYBYTES
+const HASH_SEED = b4a.alloc(sodium.crypto_shorthash_BYTES)
+
 class SystemSnapshot {
   constructor(bee) {
     this.bee = bee
@@ -41,6 +47,7 @@ module.exports = class Systembee {
     this.bee = new Hyperbee(store, opts)
     this.view = null
     this.heads = []
+    this.hash = null
     this.migration = null
     this.version = 0
     this.timestamp = 0
@@ -142,6 +149,7 @@ module.exports = class Systembee {
     this.view = info ? info.view : EMPTY_HEAD
     this.migration = null
     this.heads = info ? info.heads : []
+    this.hash = info ? info.hash : null
     this.timestamp = info ? info.timestamp : 0
     this.flushes = info ? info.flushes : 0
 
@@ -224,6 +232,8 @@ module.exports = class Systembee {
     this.heads.push({ key: node.key, length: node.length })
     if (node.timestamp > this.timestamp) this.timestamp = node.timestamp // TODO: support smoothing
 
+    this.hash = advanceHash(this.hash, node)
+
     // resolved sort weight, re-stamped on every reapplication - this is what
     // topo reads back out of the changes stream
     this.update(node.key, node.length, node.weight, -1, false, false, false, node.timestamp, null)
@@ -443,7 +453,8 @@ module.exports = class Systembee {
       timestamp: this.timestamp,
       flushes: this.flushes,
       indexers: null, // legacy
-      pending: this.promotions.digest
+      pending: this.promotions.digest,
+      hash: this.hash
     }
 
     w.tryPut(INFO_KEY, encoding.encodeSystemInfo(info))
@@ -488,3 +499,24 @@ module.exports = class Systembee {
     return this.promotions.hint(key, opts)
   }
 }
+
+function advanceHash(prev, node) {
+  const state = { start: 0, end: 0, buffer: null }
+
+  HASH.preencode(state, prev || HASH_SEED)
+  c.fixed32.preencode(state, node.key)
+  c.uint.preencode(state, node.length)
+  c.uint.preencode(state, node.weight)
+
+  state.buffer = b4a.allocUnsafe(state.end)
+
+  HASH.encode(state, prev || HASH_SEED)
+  c.fixed32.encode(state, node.key)
+  c.uint.encode(state, node.length)
+  c.uint.encode(state, node.weight)
+
+  const hash = b4a.allocUnsafe(sodium.crypto_shorthash_BYTES)
+  sodium.crypto_shorthash(hash, state.buffer, HASH_KEY)
+
+  return hash
+}

package/lib/writers.js

@@ -86,8 +86,17 @@ class ActiveWriters {
     return this.localWriter.getLatest()
   }
 
-  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null) {
-    return this.localWriter.append(value, timestamp, batch, links, optimistic, witness, approvals)
+  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
+    return this.localWriter.append(
+      value,
+      timestamp,
+      batch,
+      links,
+      optimistic,
+      witness,
+      approvals,
+      hash
+    )
   }
 
   async rotateLocalWriter(core) {
@@ -134,10 +143,20 @@ class ActiveWriters {
     // peeked batches were read against the pre-reboot system, drop them
     this._pendingBuffer.clear()
 
-    if (this.localWriter && !this.localWriter.isClosed) {
-      await this.updateLocalState()
+    const local = this.localWriter && !this.localWriter.isClosed ? this.localWriter : null
+
+    const info = []
+    for (const w of this.active.values()) {
+      if (w.isFrozen && w !== this.localWriter) continue
+      info.push(w.system.get(w.core.key))
     }
 
+    if (local) info.push(local.system.get(local.core.key))
+
+    await Promise.all(info)
+
+    if (local) await this.updateLocalState()
+
     for (const writer of this.active.values()) {
       if (writer.isFrozen && writer !== this.localWriter) await writer.detachAndClose()
       else await writer.reset()
@@ -683,7 +702,7 @@ class Writer {
     return this.pending !== null
   }
 
-  append(value, timestamp, batch, links, optimistic, witness, approvals = null) {
+  append(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
     if (this.pending === null) this.pending = []
 
     const oplog = {
@@ -696,7 +715,8 @@ class Writer {
       witness: witness || null,
       approvals,
       optimistic: optimistic && !this.writers.writable,
-      value
+      value,
+      hash
     }
 
     const node = createNode(this.core, this.appendLength++, -1, oplog)
@@ -773,7 +793,8 @@ function createNode(core, length, weight, oplog) {
     witness: oplog.witness || null,
     approvals: oplog.approvals || null,
     optimistic: oplog.optimistic,
-    value: oplog.value
+    value: oplog.value,
+    hash: oplog.hash || null
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.5.5",
+  "version": "2.7.0",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -35,7 +35,8 @@
     "ready-guard": "^1.0.0",
     "ready-resource": "^1.2.0",
     "resolve-reject-promise": "^1.1.0",
-    "safety-catch": "^1.0.3"
+    "safety-catch": "^1.0.3",
+    "sodium-native": "^5.1.0"
   },
   "devDependencies": {
     "brittle": "^4.1.0",