autobee 2.11.0 → 2.11.2

package/index.js

@@ -6,7 +6,6 @@ const Hyperbee = require('hyperbee2')
 const ID = require('hypercore-id-encoding')
 const rrp = require('resolve-reject-promise')
 const Signal = require('signal-promise')
-const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('autobee-encryption')
 const AutobeeWakeup = require('autobee-wakeup')
 const Hypercore = require('hypercore')
 const crypto = require('hypercore-crypto')
@@ -16,6 +15,7 @@ const boot = require('./lib/boot.js')
 const { DEFAULT_MANIFEST_VERSION } = require('./lib/constants.js')
 const { resolveWeight, currentWeight } = require('./lib/witness.js')
 const encoding = require('./lib/encoding.js')
+const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('./lib/encryption.js')
 const FastForward = require('./lib/fast-forward.js')
 const System = require('./lib/system.js')
 const ApplyCalls = require('./lib/apply-calls.js')
@@ -684,18 +684,14 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async compactMaybe() {
-    if (!this.writers.writable) return 0
+    if (await this._isReindexed()) return
 
-    if (await this._isReindexed()) return 0
+    await this._reindexBee(this._workingBee)
+    this.bee.move(this._workingBee.head())
 
-    const view = await this._reindexBee(this._workingBee)
-    if (view > 0) this.bee.move(this._workingBee.head())
+    await this._reindexBee(this.system.bee)
 
-    const system = await this._reindexBee(this.system.bee)
-
-    if (view + system > 0) this._reindexed = true
-
-    return view + system
+    this._reindexed = true
   }
 
   async _isReindexed() {
@@ -711,11 +707,15 @@ module.exports = class Autobee extends ReadyResource {
   async _reindexBee(bee) {
     const head = bee.head()
     const local = bee.context.local
-    if (head === null) return 0
-    if ((await this._shouldReindex(local.key)) && local.length > 0) return 0
-    if (!(await this._shouldReindex(head.key))) return 0
+    if (head === null) return
+    if (!(await this._shouldReindex(head.key))) return
+
+    // a non-empty local core is a completed reindex whose head was never stored
+    if (local.length === 0) {
+      await bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
+    }
 
-    return bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
+    bee.move({ key: local.key, length: local.length })
   }
 
   async _shouldReindex(key, { unknown = false, length = 0, timeout = 0 } = {}) {
@@ -790,6 +790,7 @@ module.exports = class Autobee extends ReadyResource {
 
           if (this.fastForwardTo !== null) {
             await this._applyFastForward()
+            await this.compactMaybe()
             if (changes) changes.track()
             this._needsUpdate = false
             break // revaluate conditions...

package/lib/encoding.js

@@ -1,10 +1,10 @@
 const b4a = require('b4a')
 const c = require('compact-encoding')
 const crypto = require('hypercore-crypto')
-const { AutobeeEncryption, WriterEncryption } = require('autobee-encryption')
 
 const { getEncoding } = require('../encoding/spec/autobee')
-const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants')
+const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants.js')
+const { AutobeeEncryption, WriterEncryption } = require('./encryption.js')
 
 const Oplog = getEncoding('@autobee/oplog')
 const Link = getEncoding('@autobee/link')

package/lib/encryption.js

@@ -0,0 +1,259 @@
+const HypercoreEncryption = require('hypercore/lib/default-encryption.js')
+const sodium = require('sodium-native')
+const crypto = require('hypercore-crypto')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+
+const { DEFAULT_MANIFEST_VERSION } = require('./constants.js')
+
+const ManifestData = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.version)
+    state.end++ // max flag is 2 so always one byte
+
+    if (m.legacyBlocks) c.uint.preencode(state, m.legacyBlocks)
+    if (m.namespace) c.fixed32.preencode(state, m.namespace)
+  },
+  encode(state, m) {
+    const flags = (m.legacyBlocks ? 1 : 0) | (m.namespace ? 2 : 0)
+
+    c.uint.encode(state, m.version)
+    c.uint.encode(state, flags)
+
+    if (m.legacyBlocks) c.uint.encode(state, m.legacyBlocks)
+    if (m.namespace) c.fixed32.encode(state, m.namespace)
+  },
+  decode(state) {
+    const r0 = c.uint.decode(state)
+    const flags = c.uint.decode(state)
+
+    return {
+      version: r0,
+      legacyBlocks: (flags & 1) !== 0 ? c.uint.decode(state) : 0,
+      namespace: (flags & 2) !== 0 ? c.fixed32.decode(state) : null
+    }
+  }
+}
+
+const [, NS_VIEW_BLOCK_KEY, NS_HASH_KEY] = crypto.namespace('autobase', 4)
+
+const [GENESIS_ENTROPY] = crypto.namespace('autobase/entropy', 2)
+
+const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
+const hash = nonce.subarray(0, sodium.crypto_generichash_BYTES_MIN)
+
+class AutobeeEncryption {
+  static PADDING = 8
+
+  constructor(auto) {
+    this.auto = auto
+  }
+
+  padding() {
+    return AutobeeEncryption.PADDING
+  }
+
+  isCompat(ctx) {
+    return ctx.manifest.version <= 1
+  }
+
+  async getKeys(id, ctx) {
+    if (!this.auto.encryptionKey) return null
+
+    const entropy = GENESIS_ENTROPY
+    const block = this.blockKey(entropy, ctx)
+    const hash = crypto.hash([NS_HASH_KEY, block])
+
+    return {
+      id,
+      block,
+      hash
+    }
+  }
+
+  blockKey(entropy, ctx) {
+    return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
+  }
+
+  compatKeys() {
+    throw new Error('Compatibility method is not specified')
+  }
+
+  async encrypt(index, block, fork, ctx) {
+    // writer cores are long lasting, but view cores should be up to date
+    if (this instanceof ViewEncryption && ctx.manifest.version < DEFAULT_MANIFEST_VERSION) {
+      throw new Error(`Unexpected manifest: version < ${DEFAULT_MANIFEST_VERSION}`)
+    }
+
+    if (this.isCompat(ctx, index)) {
+      const compat = this.compatKeys(ctx)
+      return HypercoreEncryption.encrypt(index, block, fork, compat.block, compat.blinding)
+    }
+
+    const keys = await this.getKeys(0, ctx)
+
+    encryptBlock(index, block, keys.id, keys.block, keys.hash)
+  }
+
+  async decrypt(index, block, ctx) {
+    if (this.isCompat(ctx, index)) {
+      return HypercoreEncryption.decrypt(index, block, this.compatKeys(ctx).block)
+    }
+
+    const padding = block.subarray(0, AutobeeEncryption.PADDING)
+    block = block.subarray(AutobeeEncryption.PADDING)
+
+    const type = padding[0]
+    switch (type) {
+      case 0:
+        return block // unencrypted
+
+      case 1:
+        break
+
+      default:
+        throw new Error('Unrecognised encryption type')
+    }
+
+    const id = c.uint32.decode({ start: 4, end: 8, buffer: padding })
+
+    const keys = await this.getKeys(id, ctx)
+
+    c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+    nonce.set(padding, 8, 16)
+
+    // Decrypt the block using the full nonce
+    decrypt(block, nonce, keys.block)
+  }
+
+  static encryptAnchor(block, bootstrap, encryptionKey, namespace) {
+    const entropy = GENESIS_ENTROPY
+
+    const blockKey = getBlockKey(bootstrap, encryptionKey, entropy, namespace)
+    const hashKey = crypto.hash([NS_HASH_KEY, block])
+
+    encryptBlock(0, block, 0, blockKey, hashKey)
+  }
+
+  static async setSystemEncryption(bootstrap, encryptionKey, core, opts) {
+    await core.ready()
+    if (core.manifest.version === 1) {
+      const key = getCompatBlockKey(bootstrap, encryptionKey, '_system')
+      core.setEncryptionKey(key, { block: true })
+      return
+    }
+
+    const info = {
+      key: bootstrap,
+      encryptionKey
+    }
+
+    return core.setEncryption(new ViewEncryption(info, '_system'))
+  }
+
+  static getSystemEncryption(bootstrap, encryptionKey) {
+    return AutobeeEncryption.getViewEncryption(bootstrap, encryptionKey, '_system')
+  }
+
+  static getViewEncryption(bootstrap, encryptionKey, name) {
+    const info = { key: bootstrap, encryptionKey }
+    return new ViewEncryption(info, name)
+  }
+
+  static getWriterEncryption(bootstrap, encryptionKey) {
+    return new WriterEncryption({ key: bootstrap, encryptionKey })
+  }
+}
+
+class ViewEncryption extends AutobeeEncryption {
+  constructor(auto, name) {
+    super(auto)
+    this.name = name
+  }
+
+  compatKeys() {
+    const block = getCompatBlockKey(this.auto.key, this.auto.encryptionKey, this.name)
+    return {
+      block,
+      blinding: crypto.hash(block)
+    }
+  }
+
+  blockKey(entropy, ctx) {
+    // compat encryption
+    if (ctx.manifest.version < DEFAULT_MANIFEST_VERSION) {
+      return getCompatBlockKey(this.auto.key, entropy, this.name)
+    }
+
+    return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
+  }
+}
+
+class WriterEncryption extends AutobeeEncryption {
+  compatKeys(ctx) {
+    return HypercoreEncryption.deriveKeys(this.auto.encryptionKey, ctx.key)
+  }
+
+  blockKey(entropy, ctx) {
+    // used for anchors
+    if (ctx.manifest.userData) {
+      const userData = c.decode(ManifestData, ctx.manifest.userData)
+      if (userData.namespace !== null) {
+        const b = getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, userData.namespace)
+        return b
+      }
+    }
+
+    return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
+  }
+}
+
+module.exports = {
+  AutobeeEncryption,
+  WriterEncryption,
+  ViewEncryption
+}
+
+function encrypt(block, nonce, key) {
+  sodium.crypto_stream_xor(block, block, nonce, key)
+}
+
+function decrypt(block, nonce, key) {
+  return encrypt(block, nonce, key) // symmetric
+}
+
+function getBlockKey(bootstrap, encryptionKey, entropy, hypercoreKey) {
+  return (
+    encryptionKey &&
+    crypto.hash([NS_VIEW_BLOCK_KEY, bootstrap, encryptionKey, entropy, hypercoreKey])
+  )
+}
+
+function getCompatBlockKey(bootstrap, encryptionKey, name) {
+  if (typeof name === 'string') return getCompatBlockKey(bootstrap, encryptionKey, b4a.from(name))
+  return encryptionKey && crypto.hash([NS_VIEW_BLOCK_KEY, bootstrap, encryptionKey, name])
+}
+
+function blockhash(block, padding, hashKey) {
+  sodium.crypto_generichash(hash, block, hashKey)
+  padding.set(hash.subarray(0, 8)) // copy first 8 bytes of hash
+  hash.fill(0) // clear nonce buffer
+}
+
+function encryptBlock(index, block, id, blockKey, hashKey) {
+  const padding = block.subarray(0, AutobeeEncryption.PADDING)
+  block = block.subarray(AutobeeEncryption.PADDING)
+
+  blockhash(block, padding, hashKey)
+  c.uint32.encode({ start: 4, end: 8, buffer: padding }, id)
+
+  c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
+
+  padding[0] = 1 // version in plaintext
+
+  nonce.set(padding, 8, 16)
+
+  // The combination of index, key id, fork id and block hash is very likely
+  // to be unique for a given Hypercore and therefore our nonce is suitable
+  encrypt(block, nonce, blockKey)
+}

package/lib/migrations.js

@@ -1,9 +1,9 @@
 const c = require('compact-encoding')
 const b4a = require('b4a')
 const crypto = require('hypercore-crypto')
-const { AutobeeEncryption } = require('autobee-encryption')
 const { decodeBlock } = require('hyperbee2/lib/encoding.js')
 
+const { AutobeeEncryption } = require('./encryption.js')
 const encoding = require('./encoding.js')
 const { assert, bail } = require('./asserts.js')
 

package/lib/writers.js

@@ -3,7 +3,7 @@ const encoding = require('./encoding.js')
 const Triggers = require('./triggers.js')
 const { LEGACY_OPLOG_VERSION } = require('./constants.js')
 const { resolveWeight } = require('./witness.js')
-const { WriterEncryption } = require('autobee-encryption')
+const { WriterEncryption } = require('./encryption.js')
 const ReadWriteLock = require('read-write-mutexify')
 
 const WRITER_PREFETCH = 10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.11.0",
+  "version": "2.11.2",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -23,11 +23,10 @@
     "test:generate": "brittle -r test/all.js test/*.js"
   },
   "dependencies": {
-    "autobee-encryption": "^0.1.2",
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
-    "compact-encoding": "^3.3.0",
-    "hyperbee2": "^2.17.0",
+    "compact-encoding": "^3.4.0",
+    "hyperbee2": "^2.17.1",
     "hypercore": "^11.37.0",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.26.2",