autobee 2.0.0-rc.8 → 2.0.0

package/README.md

@@ -78,7 +78,10 @@ Options:
   encryptionKey: Buffer,         // 32-byte key to encrypt all data at rest
   encrypted: false,              // set true if using encryptionKey
   keyPair: { publicKey, secretKey }, // custom signing key pair for the local writer
-  optimistic: true               // allow optimistic writes from unknown writers
+  optimistic: true,              // allow optimistic writes from unknown writers
+  isTrusted (key, reference) {}, // do we trust this writer, see Fast-forward
+  mostRecentTrusted (target, reference) {}, // the head we vouch for, see Fast-forward
+  fastForward: {}                // see Fast-forward
 }
 ```
 
@@ -222,6 +225,50 @@ Create an anchor node. Returns `{ key, length }`. Anchors are used to create a v
 
 `true` if the system has not yet processed any nodes. Use this to bootstrap the first writer.
 
+### Fast-forward
+
+Fast-forward only ever deals in oplog heads — `{ key, length }` of a writer's core, never a system head.
+
+Each flush stamps the head you vouch for into your own oplog, and peers read those stamps out of the writers they wake up on, so trust travels with the log.
+
+#### `isTrusted(key, reference)`
+
+Return whether `key` is a writer you trust, judged against the `reference` view.
+
+Positive answers are cached until an undo rewinds the view, and the default is `true` unless you supply `mostRecentTrusted`, in which case it is `false`.
+
+#### `mostRecentTrusted(target, reference)`
+
+Return the oplog head you most recently vouched for, given the `target` view being considered.
+
+Called at flush time to stamp your own oplog (with your view as `target` and a `null` reference), and again per candidate during discovery.
+
+#### `fastForward.boot`
+
+```js
+{
+  head: { key, length },       // oplog head to boot from
+  legacy: { key, length },     // pre-2.0 pointer, see below
+  bootCondition (target, reference) {} // optional gate on the view we would land on
+}
+```
+
+Pass one of `head` or `legacy`, not both.
+
+Without `bootCondition` this is a single attempt that gives up if the head cannot be read; with one it parks until the condition is satisfied, which today means indefinitely if it never is.
+
+`legacy` is for records written before boot heads were oplog heads: the key is a _system_ head and a `0` length is resolved from the core. It boots ungated - `bootCondition` does not apply - and will be removed, so don't reach for it. A bare `{ key, length }` in place of the whole struct means the same thing, older still.
+
+#### `fastForward.conservative`
+
+Defaults to `true`: only fast-forward onto a head a connected peer can serve whole.
+
+The check covers the oplog head only, not the system and view cores the fast-forward then reads.
+
+#### `await db.moveTo(head)`
+
+Fast-forward onto an oplog head, ignoring the usual distance and conservative checks. Resolves `{ to, from }`.
+
 ### Encryption
 
 Pass an `encryptionKey` to encrypt all writer cores and the view at rest.

package/encoding/compat.js

@@ -169,7 +169,7 @@ module.exports = function buildAutobaseSchema(schema, DIR) {
       },
       {
         name: 'value',
-        type: 'buffer',
+        type: 'optionalBuffer',
         required: true
       }
     ]

package/encoding/legacy.js

@@ -153,18 +153,18 @@ const Node = {
   preencode(state, m) {
     Clock.preencode(state, m.heads)
     c.uint.preencode(state, m.batch)
-    c.buffer.preencode(state, m.value)
+    c.optionalBuffer.preencode(state, m.value)
   },
   encode(state, m) {
     Clock.encode(state, m.heads)
     c.uint.encode(state, m.batch)
-    c.buffer.encode(state, m.value)
+    c.optionalBuffer.encode(state, m.value)
   },
   decode(state, m) {
     return {
       heads: Clock.decode(state),
       batch: c.uint.decode(state),
-      value: c.buffer.decode(state)
+      value: c.optionalBuffer.decode(state)
     }
   }
 }
@@ -280,8 +280,8 @@ const SystemWriterV0 = {
 
     return {
       version: 0,
-      isIndexer: flags & 1,
-      isRemoved: flags & 2,
+      isIndexer: (flags & 1) !== 0,
+      isRemoved: (flags & 2) !== 0,
       length: c.uint.decode(state)
     }
   }
@@ -300,11 +300,35 @@ function infoLegacyMap(info) {
 }
 
 function memberLegacyMap(m) {
+  const weight = m.isIndexer ? 2 : 1
+
   return {
     version: 0,
     isRemoved: m.isRemoved,
-    weight: m.isIndexer ? 2 : 1,
-    length: m.length
+    isOplog: false,
+    isGenesis: false,
+    weight,
+    maxWeight: weight,
+    length: m.length,
+    timestamp: 0
+  }
+}
+
+// a v3 node's witness is a signed attestation, which confers weight through a
+// mechanism that no longer exists - it maps to null, so the node sorts at the
+// standing its record already carries instead of failing to decode
+function oplogLegacyMap(m) {
+  return {
+    version: m.version,
+    timestamp: m.timestamp,
+    links: m.links,
+    batch: m.batch,
+    views: m.views,
+    trusted: m.trusted,
+    witness: null,
+    approvals: null,
+    optimistic: m.optimistic,
+    value: m.value
   }
 }
 
@@ -314,5 +338,6 @@ module.exports = {
   OplogMessageV1,
   SystemWriterV0,
   infoLegacyMap,
-  memberLegacyMap
+  memberLegacyMap,
+  oplogLegacyMap
 }

package/encoding/spec/autobee/index.js

@@ -12,6 +12,36 @@ const VERSION = 1
 // eslint-disable-next-line no-unused-vars
 let version = VERSION
 
+// array of bools, packed as a bitfield
+const boolArray = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.length)
+    state.end += Math.ceil(m.length / 8)
+  },
+  encode(state, m) {
+    c.uint.encode(state, m.length)
+    for (let i = 0; i < m.length; i += 8) {
+      let byte = 0
+      for (let j = 0; j < 8 && i + j < m.length; j++) {
+        if (m[i + j]) byte |= 1 << j
+      }
+      state.buffer[state.start++] = byte
+    }
+  },
+  decode(state) {
+    const n = c.uint.decode(state)
+    if (state.end - state.start < Math.ceil(n / 8)) throw new Error('Out of bounds')
+    const m = new Array(n)
+    for (let i = 0; i < n; i += 8) {
+      const byte = state.buffer[state.start++]
+      for (let j = 0; j < 8 && i + j < n; j++) {
+        m[i + j] = (byte & (1 << j)) !== 0
+      }
+    }
+    return m
+  }
+}
+
 // @autobase-compat/checkout
 const encoding0 = {
   preencode(state, m) {
@@ -97,8 +127,9 @@ const encoding4 = {
 // @autobase-compat/boot-record
 const encoding5 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
         encoding3.preencode(state, m)
         break
@@ -112,8 +143,9 @@ const encoding5 = {
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
         encoding3.encode(state, m)
         break
@@ -236,17 +268,17 @@ const encoding9 = {
   preencode(state, m) {
     encoding1.preencode(state, m.heads)
     c.uint.preencode(state, m.batch)
-    c.buffer.preencode(state, m.value)
+    c.optionalBuffer.preencode(state, m.value)
   },
   encode(state, m) {
     encoding1.encode(state, m.heads)
     c.uint.encode(state, m.batch)
-    c.buffer.encode(state, m.value)
+    c.optionalBuffer.encode(state, m.value)
   },
   decode(state) {
     const r0 = encoding1.decode(state)
     const r1 = c.uint.decode(state)
-    const r2 = c.buffer.decode(state)
+    const r2 = c.optionalBuffer.decode(state)
 
     return {
       heads: r0,
@@ -452,6 +484,9 @@ const encoding16 = {
 
 const encoding17 = external0.SystemWriterV0
 
+// @autobee/system-info-v3.pending
+const encoding18_5 = boolArray
+
 // @autobee/system-info-v3
 const encoding18 = {
   preencode(state, m) {
@@ -459,12 +494,13 @@ const encoding18 = {
     c.uint.preencode(state, m.flushes)
     encoding22.preencode(state, m.view)
     encoding18_3.preencode(state, m.heads)
-    state.end++ // max flag is 1 so always one byte
+    state.end++ // max flag is 2 so always one byte
 
     if (m.indexers) encoding18_4.preencode(state, m.indexers)
+    if (m.pending) encoding18_5.preencode(state, m.pending)
   },
   encode(state, m) {
-    const flags = m.indexers ? 1 : 0
+    const flags = (m.indexers ? 1 : 0) | (m.pending ? 2 : 0)
 
     c.uint.encode(state, m.timestamp)
     c.uint.encode(state, m.flushes)
@@ -473,6 +509,7 @@ const encoding18 = {
     c.uint.encode(state, flags)
 
     if (m.indexers) encoding18_4.encode(state, m.indexers)
+    if (m.pending) encoding18_5.encode(state, m.pending)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -488,7 +525,8 @@ const encoding18 = {
       flushes: r1,
       view: r2,
       heads: r3,
-      indexers: (flags & 1) !== 0 ? encoding18_4.decode(state) : null
+      indexers: (flags & 1) !== 0 ? encoding18_4.decode(state) : null,
+      pending: (flags & 2) !== 0 ? encoding18_5.decode(state) : null
     }
   }
 }
@@ -496,8 +534,9 @@ const encoding18 = {
 // @autobee/system-info
 const encoding19 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
       case 1:
         encoding15.preencode(state, m)
@@ -513,8 +552,9 @@ const encoding19 = {
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
       case 1:
         encoding15.encode(state, m)
@@ -603,8 +643,9 @@ const encoding20 = {
 // @autobee/system-writer
 const encoding21 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 5
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
       case 1:
       case 2:
@@ -614,13 +655,17 @@ const encoding21 = {
       case 4:
         encoding20.preencode(state, m)
         break
+      case 5:
+        encoding34.preencode(state, m)
+        break
       default:
         throw new Error('Unsupported version')
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 5
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
       case 1:
       case 2:
@@ -630,6 +675,9 @@ const encoding21 = {
       case 4:
         encoding20.encode(state, m)
         break
+      case 5:
+        encoding34.encode(state, m)
+        break
       default:
         throw new Error('Unsupported version')
     }
@@ -651,6 +699,10 @@ const encoding21 = {
         const decoded = encoding20.decode(state)
         return decoded
       }
+      case 5: {
+        const decoded = encoding34.decode(state)
+        return decoded
+      }
       default:
         throw new Error('Unsupported version')
     }
@@ -850,15 +902,28 @@ const encoding29_7 = c.array(c.frame(encoding28))
 // @autobee/oplog-message-v3
 const encoding29 = {
   preencode(state, m) {
+    let flags =
+      (m.batch ? 1 : 0) |
+      (m.views ? 2 : 0) |
+      (m.optimistic ? 8 : 0) |
+      (m.value ? 16 : 0) |
+      (m.witness ? 32 : 0) |
+      (m.attestations ? 64 : 0) |
+      (m.trusted ? 128 : 0)
+    if (m.views) {
+      flags |= m.views.view ? 4 : 0
+    }
+
     c.uint.preencode(state, m.timestamp)
     encoding29_1.preencode(state, m.links)
-    state.end++ // max flag is 64 so always one byte
+    c.uint.preencode(state, flags)
 
     if (m.batch) encoding24.preencode(state, m.batch)
     if (m.views) encoding25_inline.preencode(state, m.views)
     if (m.value) c.buffer.preencode(state, m.value)
     if (m.witness) encoding29_6.preencode(state, m.witness)
     if (m.attestations) encoding29_7.preencode(state, m.attestations)
+    if (m.trusted) encoding29_8.preencode(state, m.trusted)
   },
   encode(state, m) {
     let flags =
@@ -867,7 +932,8 @@ const encoding29 = {
       (m.optimistic ? 8 : 0) |
       (m.value ? 16 : 0) |
       (m.witness ? 32 : 0) |
-      (m.attestations ? 64 : 0)
+      (m.attestations ? 64 : 0) |
+      (m.trusted ? 128 : 0)
     if (m.views) {
       flags |= m.views.view ? 4 : 0
     }
@@ -881,6 +947,7 @@ const encoding29 = {
     if (m.value) c.buffer.encode(state, m.value)
     if (m.witness) encoding29_6.encode(state, m.witness)
     if (m.attestations) encoding29_7.encode(state, m.attestations)
+    if (m.trusted) encoding29_8.encode(state, m.trusted)
   },
   decode(state) {
     const v = c.uint.decode(state)
@@ -897,7 +964,8 @@ const encoding29 = {
       optimistic: (flags & 8) !== 0,
       value: (flags & 16) !== 0 ? c.buffer.decode(state) : null,
       witness: (flags & 32) !== 0 ? encoding29_6.decode(state) : null,
-      attestations: (flags & 64) !== 0 ? encoding29_7.decode(state) : null
+      attestations: (flags & 64) !== 0 ? encoding29_7.decode(state) : null,
+      trusted: (flags & 128) !== 0 ? encoding29_8.decode(state) : null
     }
   }
 }
@@ -905,8 +973,9 @@ const encoding29 = {
 // @autobee/oplog
 const encoding30 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 4
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
         encoding12.preencode(state, m)
         break
@@ -919,13 +988,17 @@ const encoding30 = {
       case 3:
         encoding29.preencode(state, m)
         break
+      case 4:
+        encoding37.preencode(state, m)
+        break
       default:
         throw new Error('Unsupported version')
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 4
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
         encoding12.encode(state, m)
         break
@@ -938,6 +1011,9 @@ const encoding30 = {
       case 3:
         encoding29.encode(state, m)
         break
+      case 4:
+        encoding37.encode(state, m)
+        break
       default:
         throw new Error('Unsupported version')
     }
@@ -961,6 +1037,11 @@ const encoding30 = {
       }
       case 3: {
         const decoded = encoding29.decode(state)
+        const map = external0.oplogLegacyMap
+        return map(decoded)
+      }
+      case 4: {
+        const decoded = encoding37.decode(state)
         return decoded
       }
       default:
@@ -999,12 +1080,206 @@ const encoding31 = {
   }
 }
 
+// @autobee/trusted-head
+const encoding32 = {
+  preencode(state, m) {
+    c.fixed32.preencode(state, m.key)
+    c.uint.preencode(state, m.length)
+    c.uint.preencode(state, m.flushes)
+  },
+  encode(state, m) {
+    c.fixed32.encode(state, m.key)
+    c.uint.encode(state, m.length)
+    c.uint.encode(state, m.flushes)
+  },
+  decode(state) {
+    const r0 = c.fixed32.decode(state)
+    const r1 = c.uint.decode(state)
+    const r2 = c.uint.decode(state)
+
+    return {
+      key: r0,
+      length: r1,
+      flushes: r2
+    }
+  }
+}
+
+// @autobee/migrated-head
+const encoding33 = {
+  preencode(state, m) {
+    encoding22.preencode(state, m.system)
+    state.end++ // flags are fixed size
+
+    if (m.view) encoding22.preencode(state, m.view)
+  },
+  encode(state, m) {
+    const flags = m.view ? 1 : 0
+
+    encoding22.encode(state, m.system)
+    c.uint8.encode(state, flags)
+
+    if (m.view) encoding22.encode(state, m.view)
+  },
+  decode(state) {
+    const r0 = encoding22.decode(state)
+    const flags = c.uint8.decode(state)
+
+    return {
+      system: r0,
+      view: (flags & 1) !== 0 ? encoding22.decode(state) : null
+    }
+  }
+}
+
+// @autobee/system-writer-v5
+const encoding34 = {
+  preencode(state, m) {
+    state.end++ // max flag is 4 so always one byte
+    c.uint.preencode(state, m.weight)
+    c.uint.preencode(state, m.maxWeight)
+    c.uint.preencode(state, m.length)
+    c.uint.preencode(state, m.timestamp)
+  },
+  encode(state, m) {
+    const flags = (m.isRemoved ? 1 : 0) | (m.isOplog ? 2 : 0) | (m.isGenesis ? 4 : 0)
+
+    c.uint.encode(state, flags)
+    c.uint.encode(state, m.weight)
+    c.uint.encode(state, m.maxWeight)
+    c.uint.encode(state, m.length)
+    c.uint.encode(state, m.timestamp)
+  },
+  decode(state) {
+    const v = c.uint.decode(state)
+    const flags = c.uint.decode(state)
+
+    return {
+      version: v,
+      isRemoved: (flags & 1) !== 0,
+      isOplog: (flags & 2) !== 0,
+      isGenesis: (flags & 4) !== 0,
+      weight: c.uint.decode(state),
+      maxWeight: c.uint.decode(state),
+      length: c.uint.decode(state),
+      timestamp: c.uint.decode(state)
+    }
+  }
+}
+
+// @autobee/grant-witness
+const encoding35 = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.weight)
+    encoding22.preencode(state, m.link)
+  },
+  encode(state, m) {
+    c.uint.encode(state, m.weight)
+    encoding22.encode(state, m.link)
+  },
+  decode(state) {
+    const r0 = c.uint.decode(state)
+    const r1 = encoding22.decode(state)
+
+    return {
+      weight: r0,
+      link: r1
+    }
+  }
+}
+
+// @autobee/approval
+const encoding36 = {
+  preencode(state, m) {
+    c.fixed32.preencode(state, m.key)
+    c.uint.preencode(state, m.weight)
+  },
+  encode(state, m) {
+    c.fixed32.encode(state, m.key)
+    c.uint.encode(state, m.weight)
+  },
+  decode(state) {
+    const r0 = c.fixed32.decode(state)
+    const r1 = c.uint.decode(state)
+
+    return {
+      key: r0,
+      weight: r1
+    }
+  }
+}
+
+// @autobee/oplog-message-v4.approvals
+const encoding37_6 = c.array(encoding36)
+
+// @autobee/oplog-message-v4
+const encoding37 = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.timestamp)
+    encoding37_1.preencode(state, m.links)
+    state.end++ // max flag is 64 so always one byte
+
+    if (m.batch) encoding24.preencode(state, m.batch)
+    if (m.views) encoding25.preencode(state, m.views)
+    if (m.trusted) encoding37_4.preencode(state, m.trusted)
+    if (m.witness) encoding35.preencode(state, m.witness)
+    if (m.approvals) encoding37_6.preencode(state, m.approvals)
+    if (m.value) c.buffer.preencode(state, m.value)
+  },
+  encode(state, m) {
+    const flags =
+      (m.batch ? 1 : 0) |
+      (m.views ? 2 : 0) |
+      (m.trusted ? 4 : 0) |
+      (m.witness ? 8 : 0) |
+      (m.approvals ? 16 : 0) |
+      (m.optimistic ? 32 : 0) |
+      (m.value ? 64 : 0)
+
+    c.uint.encode(state, m.timestamp)
+    encoding37_1.encode(state, m.links)
+    c.uint.encode(state, flags)
+
+    if (m.batch) encoding24.encode(state, m.batch)
+    if (m.views) encoding25.encode(state, m.views)
+    if (m.trusted) encoding37_4.encode(state, m.trusted)
+    if (m.witness) encoding35.encode(state, m.witness)
+    if (m.approvals) encoding37_6.encode(state, m.approvals)
+    if (m.value) c.buffer.encode(state, m.value)
+  },
+  decode(state) {
+    const v = c.uint.decode(state)
+    const r0 = c.uint.decode(state)
+    const r1 = encoding37_1.decode(state)
+    const flags = c.uint.decode(state)
+
+    return {
+      version: v,
+      timestamp: r0,
+      links: r1,
+      batch: (flags & 1) !== 0 ? encoding24.decode(state) : null,
+      views: (flags & 2) !== 0 ? encoding25.decode(state) : null,
+      trusted: (flags & 4) !== 0 ? encoding37_4.decode(state) : null,
+      witness: (flags & 8) !== 0 ? encoding35.decode(state) : null,
+      approvals: (flags & 16) !== 0 ? encoding37_6.decode(state) : null,
+      optimistic: (flags & 32) !== 0,
+      value: (flags & 64) !== 0 ? c.buffer.decode(state) : null
+    }
+  }
+}
+
 // @autobee/system-info-v3.heads, deferred due to recusive use
 const encoding18_3 = c.array(encoding22)
 // @autobee/system-info-v3.indexers, deferred due to recusive use
 const encoding18_4 = encoding18_3
 // @autobee/oplog-message-v3.links, deferred due to recusive use
 const encoding29_1 = encoding18_3
+// @autobee/oplog-message-v3.trusted, deferred due to recusive use
+const encoding29_8 = c.array(c.frame(encoding32))
+// @autobee/oplog-message-v4.links, deferred due to recusive use
+const encoding37_1 = encoding18_3
+// @autobee/oplog-message-v4.trusted, deferred due to recusive use
+const encoding37_4 = encoding18_3
 
 function setVersion(v) {
   version = v
@@ -1093,6 +1368,18 @@ function getEncoding(name) {
       return encoding30
     case '@autobee/manifest-data':
       return encoding31
+    case '@autobee/trusted-head':
+      return encoding32
+    case '@autobee/migrated-head':
+      return encoding33
+    case '@autobee/system-writer-v5':
+      return encoding34
+    case '@autobee/grant-witness':
+      return encoding35
+    case '@autobee/approval':
+      return encoding36
+    case '@autobee/oplog-message-v4':
+      return encoding37
     default:
       throw new Error('Encoder not found ' + name)
   }