autobee 2.6.0 → 2.7.1

package/README.md

@@ -256,13 +256,17 @@ Return the oplog head you most recently vouched for, given the `target` view bei
 
 Called at flush time to stamp your own oplog (with your view as `target` and a `null` reference), and again per candidate during discovery.
 
+During discovery `target` is a view opened for the candidate, and reads through it carry the fast-forward's timeout. Anything else the hook awaits is unbounded, so if it reaches beyond that view, put a timeout on it: the candidate waits on the hook, and so does closing it.
+
 #### `warmup(view)`
 
 Prepare a candidate `view` before the fast-forward onto it is applied.
 
 Called as soon as the candidate's view head is known, and runs alongside the block downloads the fast-forward is already doing rather than after them. Throw or reject to reject the candidate: the fast-forward is abandoned and the usual apply path catches up instead.
 
-`view` is opened and closed through the `open` and `close` handlers, so don't build or close a db of your own.
+`view` is opened and closed through the `open` and `close` handlers, so don't build or close a db of your own. Reads through it carry the fast-forward's timeout, so a warmup that needs blocks nobody serves fails the candidate instead of hanging.
+
+Only reads through `view` are bounded. Anything else the handler awaits, such as other cores or your own network calls, is up to you to time out: the fast-forward waits on the handler, and an abandoned candidate cannot finish closing until it returns.
 
 #### `fastForward.boot`
 

package/encoding/spec/autobee/index.js

@@ -494,13 +494,14 @@ const encoding18 = {
     c.uint.preencode(state, m.flushes)
     encoding22.preencode(state, m.view)
     encoding18_3.preencode(state, m.heads)
-    state.end++ // max flag is 2 so always one byte
+    state.end++ // max flag is 4 so always one byte
 
     if (m.indexers) encoding18_4.preencode(state, m.indexers)
     if (m.pending) encoding18_5.preencode(state, m.pending)
+    if (m.hash) c.fixed8.preencode(state, m.hash)
   },
   encode(state, m) {
-    const flags = (m.indexers ? 1 : 0) | (m.pending ? 2 : 0)
+    const flags = (m.indexers ? 1 : 0) | (m.pending ? 2 : 0) | (m.hash ? 4 : 0)
 
     c.uint.encode(state, m.timestamp)
     c.uint.encode(state, m.flushes)
@@ -510,6 +511,7 @@ const encoding18 = {
 
     if (m.indexers) encoding18_4.encode(state, m.indexers)
     if (m.pending) encoding18_5.encode(state, m.pending)
+    if (m.hash) c.fixed8.encode(state, m.hash)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -526,7 +528,8 @@ const encoding18 = {
       view: r2,
       heads: r3,
       indexers: (flags & 1) !== 0 ? encoding18_4.decode(state) : null,
-      pending: (flags & 2) !== 0 ? encoding18_5.decode(state) : null
+      pending: (flags & 2) !== 0 ? encoding18_5.decode(state) : null,
+      hash: (flags & 4) !== 0 ? c.fixed8.decode(state) : null
     }
   }
 }
@@ -1215,9 +1218,19 @@ const encoding37_6 = c.array(encoding36)
 // @autobee/oplog-message-v4
 const encoding37 = {
   preencode(state, m) {
+    const flags =
+      (m.batch ? 1 : 0) |
+      (m.views ? 2 : 0) |
+      (m.trusted ? 4 : 0) |
+      (m.witness ? 8 : 0) |
+      (m.approvals ? 16 : 0) |
+      (m.optimistic ? 32 : 0) |
+      (m.value ? 64 : 0) |
+      (m.hash ? 128 : 0)
+
     c.uint.preencode(state, m.timestamp)
     encoding37_1.preencode(state, m.links)
-    state.end++ // max flag is 64 so always one byte
+    c.uint.preencode(state, flags)
 
     if (m.batch) encoding24.preencode(state, m.batch)
     if (m.views) encoding25.preencode(state, m.views)
@@ -1225,6 +1238,7 @@ const encoding37 = {
     if (m.witness) encoding35.preencode(state, m.witness)
     if (m.approvals) encoding37_6.preencode(state, m.approvals)
     if (m.value) c.buffer.preencode(state, m.value)
+    if (m.hash) c.fixed8.preencode(state, m.hash)
   },
   encode(state, m) {
     const flags =
@@ -1234,7 +1248,8 @@ const encoding37 = {
       (m.witness ? 8 : 0) |
       (m.approvals ? 16 : 0) |
       (m.optimistic ? 32 : 0) |
-      (m.value ? 64 : 0)
+      (m.value ? 64 : 0) |
+      (m.hash ? 128 : 0)
 
     c.uint.encode(state, m.timestamp)
     encoding37_1.encode(state, m.links)
@@ -1246,6 +1261,7 @@ const encoding37 = {
     if (m.witness) encoding35.encode(state, m.witness)
     if (m.approvals) encoding37_6.encode(state, m.approvals)
     if (m.value) c.buffer.encode(state, m.value)
+    if (m.hash) c.fixed8.encode(state, m.hash)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -1263,7 +1279,8 @@ const encoding37 = {
       witness: (flags & 8) !== 0 ? encoding35.decode(state) : null,
       approvals: (flags & 16) !== 0 ? encoding37_6.decode(state) : null,
       optimistic: (flags & 32) !== 0,
-      value: (flags & 64) !== 0 ? c.buffer.decode(state) : null
+      value: (flags & 64) !== 0 ? c.buffer.decode(state) : null,
+      hash: (flags & 128) !== 0 ? c.fixed8.decode(state) : null
     }
   }
 }

package/encoding/spec/autobee/schema.json

@@ -434,6 +434,11 @@
           "array": true,
           "type": "bool",
           "version": 1
+        },
+        {
+          "name": "hash",
+          "type": "fixed8",
+          "version": 1
         }
       ]
     },
@@ -1018,6 +1023,11 @@
           "name": "value",
           "type": "buffer",
           "version": 1
+        },
+        {
+          "name": "hash",
+          "type": "fixed8",
+          "version": 1
         }
       ]
     }

package/index.js

@@ -437,15 +437,17 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   // the view an oplog head points at, opened and closed through the handlers
-  openViewAt(oplog) {
+  openViewAt(oplog, { timeout } = {}) {
     const v = oplog.op.views.view
     if (!v) return null
 
-    return this.openView({ key: v.key, length: v.start + v.length })
+    return this.openView({ key: v.key, length: v.start + v.length }, { timeout })
   }
 
-  openView(head) {
-    return new ApplyView(this.bee.checkout({ key: head.key, length: head.length }), this)
+  // timeout bounds every hypercore read done through the view, defaults to the main bee's
+  openView(head, { timeout } = {}) {
+    const bee = this.bee.checkout({ key: head.key, length: head.length, timeout })
+    return new ApplyView(bee, this)
   }
 
   openCore(key) {
@@ -1196,9 +1198,14 @@ module.exports = class Autobee extends ReadyResource {
     node.weight = await resolveWeight(this, node)
     for (const n of batch) n.weight = node.weight
 
-    // if (topo.isLinkingAll(node, this.system.heads)) {
-    //   return { undo: null, view: null, tip: [batch] }
-    // }
+    if (
+      node.hash &&
+      this.system.hash &&
+      b4a.equals(node.hash, this.system.hash) &&
+      topo.isLinkingAll(node, this.system.heads)
+    ) {
+      return { undo: null, view: null, tip: [batch] }
+    }
 
     const t = await topo.sort(this, batch)
 
@@ -1352,6 +1359,7 @@ module.exports = class Autobee extends ReadyResource {
     await this.local.ready()
 
     const links = this.system.getLinks(this.local.key)
+    const hash = this.system.hash
 
     // never stamp before anything we link
     const t = Math.max(this._now(), this.system.timestamp)
@@ -1379,7 +1387,8 @@ module.exports = class Autobee extends ReadyResource {
         lnk,
         optimistic,
         i === 0 ? witness : null,
-        i === 0 ? approvals : null
+        i === 0 ? approvals : null,
+        i === 0 ? hash : null
       )
       batch.push(node)
     }

package/lib/fast-forward.js

@@ -56,7 +56,9 @@ class FastForward {
       return null
     }
 
-    return new FastForward(auto, batchToHead(verified.op.views.system))
+    return new FastForward(auto, batchToHead(verified.op.views.system), {
+      timeout: timeout || DEFAULT_OP_TIMEOUT
+    })
   }
 
   static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
@@ -107,7 +109,7 @@ class FastForward {
     candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
 
     for (const res of candidates) {
-      const trusted = await FastForward.mostRecentTrusted(auto, res, reference)
+      const trusted = await FastForward.mostRecentTrusted(auto, res, reference, timeout)
       if (trusted === null) continue
       if (auto.fastForwarding || auto.fastForwardTo) return null
 
@@ -130,8 +132,8 @@ class FastForward {
     }
   }
 
-  static async mostRecentTrusted(auto, head, reference) {
-    const opened = auto.openViewAt(head)
+  static async mostRecentTrusted(auto, head, reference, timeout) {
+    const opened = auto.openViewAt(head, { timeout })
     if (opened === null) return null
 
     try {
@@ -206,7 +208,7 @@ class FastForward {
   _startWarmup() {
     if (!this.auto._warmup || !this.system.view.length) return false
 
-    this.warmupView = this.auto.openView(this.system.view)
+    this.warmupView = this.auto.openView(this.system.view, { timeout: this.timeout })
 
     // ensure thenable
     this.warmup = Promise.resolve(this.auto._warmup(this.warmupView.view))

package/lib/system.js

@@ -1,5 +1,7 @@
 const Hyperbee = require('hyperbee2')
 const Promotions = require('./promotions.js')
+const sodium = require('sodium-native')
+const c = require('compact-encoding')
 const b4a = require('b4a')
 
 const { getViewMap, getCoreManifest } = require('./migrations.js')
@@ -13,6 +15,10 @@ const INFO_LEGACY_KEY = b4a.concat([b4a.from([0, 0]), b4a.from('info')])
 const EMPTY_HEAD = { length: 0, key: null }
 const EMPTY = b4a.from([])
 
+const HASH = c.fixed(sodium.crypto_shorthash_BYTES)
+const HASH_KEY = b4a.from('autobee-orderkey') // crypto_shorthash_KEYBYTES
+const HASH_SEED = b4a.alloc(sodium.crypto_shorthash_BYTES)
+
 class SystemSnapshot {
   constructor(bee) {
     this.bee = bee
@@ -41,6 +47,7 @@ module.exports = class Systembee {
     this.bee = new Hyperbee(store, opts)
     this.view = null
     this.heads = []
+    this.hash = null
     this.migration = null
     this.version = 0
     this.timestamp = 0
@@ -142,6 +149,7 @@ module.exports = class Systembee {
     this.view = info ? info.view : EMPTY_HEAD
     this.migration = null
     this.heads = info ? info.heads : []
+    this.hash = info ? info.hash : null
     this.timestamp = info ? info.timestamp : 0
     this.flushes = info ? info.flushes : 0
 
@@ -224,6 +232,8 @@ module.exports = class Systembee {
     this.heads.push({ key: node.key, length: node.length })
     if (node.timestamp > this.timestamp) this.timestamp = node.timestamp // TODO: support smoothing
 
+    this.hash = advanceHash(this.hash, node)
+
     // resolved sort weight, re-stamped on every reapplication - this is what
     // topo reads back out of the changes stream
     this.update(node.key, node.length, node.weight, -1, false, false, false, node.timestamp, null)
@@ -443,7 +453,8 @@ module.exports = class Systembee {
       timestamp: this.timestamp,
       flushes: this.flushes,
       indexers: null, // legacy
-      pending: this.promotions.digest
+      pending: this.promotions.digest,
+      hash: this.hash
     }
 
     w.tryPut(INFO_KEY, encoding.encodeSystemInfo(info))
@@ -488,3 +499,24 @@ module.exports = class Systembee {
     return this.promotions.hint(key, opts)
   }
 }
+
+function advanceHash(prev, node) {
+  const state = { start: 0, end: 0, buffer: null }
+
+  HASH.preencode(state, prev || HASH_SEED)
+  c.fixed32.preencode(state, node.key)
+  c.uint.preencode(state, node.length)
+  c.uint.preencode(state, node.weight)
+
+  state.buffer = b4a.allocUnsafe(state.end)
+
+  HASH.encode(state, prev || HASH_SEED)
+  c.fixed32.encode(state, node.key)
+  c.uint.encode(state, node.length)
+  c.uint.encode(state, node.weight)
+
+  const hash = b4a.allocUnsafe(sodium.crypto_shorthash_BYTES)
+  sodium.crypto_shorthash(hash, state.buffer, HASH_KEY)
+
+  return hash
+}

package/lib/writers.js

@@ -86,8 +86,17 @@ class ActiveWriters {
     return this.localWriter.getLatest()
   }
 
-  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null) {
-    return this.localWriter.append(value, timestamp, batch, links, optimistic, witness, approvals)
+  appendLocal(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
+    return this.localWriter.append(
+      value,
+      timestamp,
+      batch,
+      links,
+      optimistic,
+      witness,
+      approvals,
+      hash
+    )
   }
 
   async rotateLocalWriter(core) {
@@ -693,7 +702,7 @@ class Writer {
     return this.pending !== null
   }
 
-  append(value, timestamp, batch, links, optimistic, witness, approvals = null) {
+  append(value, timestamp, batch, links, optimistic, witness, approvals = null, hash = null) {
     if (this.pending === null) this.pending = []
 
     const oplog = {
@@ -706,7 +715,8 @@ class Writer {
       witness: witness || null,
       approvals,
       optimistic: optimistic && !this.writers.writable,
-      value
+      value,
+      hash
     }
 
     const node = createNode(this.core, this.appendLength++, -1, oplog)
@@ -783,7 +793,8 @@ function createNode(core, length, weight, oplog) {
     witness: oplog.witness || null,
     approvals: oplog.approvals || null,
     optimistic: oplog.optimistic,
-    value: oplog.value
+    value: oplog.value,
+    hash: oplog.hash || null
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.6.0",
+  "version": "2.7.1",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -27,7 +27,7 @@
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
     "compact-encoding": "^3.3.0",
-    "hyperbee2": "^2.10.3",
+    "hyperbee2": "^2.16.0",
     "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.20.1",
@@ -35,7 +35,8 @@
     "ready-guard": "^1.0.0",
     "ready-resource": "^1.2.0",
     "resolve-reject-promise": "^1.1.0",
-    "safety-catch": "^1.0.3"
+    "safety-catch": "^1.0.3",
+    "sodium-native": "^5.1.0"
   },
   "devDependencies": {
     "brittle": "^4.1.0",