autobee 0.0.0 → 1.0.0

package/lib/asserts.js

@@ -0,0 +1,25 @@
+exports.heads = function (heads, node) {
+  for (const h of this.heads) {
+    if (h.key.equals(node.key)) {
+      exports.bail('Same node included twice')
+    }
+  }
+}
+
+exports.systemFlush = function (w) {
+  for (const op of w.ops) {
+    if (op.applied && op.key[0] === 1) return
+  }
+
+  exports.bail('Invalid system batch')
+}
+
+exports.assert = function (cond, msg) {
+  if (!cond) exports.bail(msg)
+}
+
+exports.bail = function (msg) {
+  const err = new Error('ERR_ASSERTION: ' + msg)
+  err.code = 'ERR_ASSERTION'
+  throw err
+}

package/lib/boot.js

@@ -0,0 +1,117 @@
+const encoding = require('./encoding.js')
+
+module.exports = async function boot(
+  corestore,
+  key,
+  { encrypt, encryptionKey, keyPair, exclusive = true } = {}
+) {
+  const result = {
+    key: null,
+    local: null,
+    bootstrap: null,
+    encryptionKey: null,
+    system: null,
+    flushes: null
+  }
+
+  const manifest = keyPair
+    ? { version: corestore.manifestVersion, signers: [{ publicKey: keyPair.publicKey }] }
+    : null
+
+  if (key) {
+    result.key = key
+
+    const bootstrap = corestore.get({ key })
+    await bootstrap.ready()
+
+    const localKey = await bootstrap.getUserData('autobee/local')
+
+    if (keyPair) {
+      result.local = corestore.get({
+        keyPair,
+        active: false,
+        exclusive,
+        manifest
+      })
+    } else {
+      if (bootstrap.writable && !localKey) {
+        result.local = bootstrap.session({
+          active: false,
+          exclusive
+        })
+      } else {
+        const local = localKey
+          ? corestore.get({
+              key: localKey,
+              active: false,
+              exclusive
+            })
+          : corestore.get({
+              name: 'local',
+              active: false,
+              exclusive
+            })
+
+        await local.ready()
+        result.local = local
+      }
+    }
+
+    result.bootstrap = bootstrap
+  } else {
+    result.local = keyPair
+      ? corestore.get({
+          keyPair,
+          manifest,
+          active: false,
+          exclusive
+        })
+      : corestore.get({
+          name: 'local',
+          active: false,
+          exclusive
+        })
+    await result.local.ready()
+
+    const key = await result.local.getUserData('referrer')
+    if (key) {
+      result.key = key
+      result.bootstrap = corestore.get({ key, active: false })
+      await result.bootstrap.ready()
+    } else {
+      result.key = result.local.key
+      result.bootstrap = result.local.session({ active: false })
+      await result.bootstrap.setUserData('autobee/local', result.local.key)
+    }
+  }
+
+  if (key || keyPair) {
+    await result.bootstrap.setUserData('referrer', result.key)
+    await result.bootstrap.setUserData('autobee/local', result.local.key)
+    await result.local.setUserData('referrer', result.key)
+  }
+
+  const [systemHead, encryptionKeyBuffer] = await Promise.all([
+    result.local.getUserData('autobee/head'),
+    result.local.getUserData('autobee/encryption')
+  ])
+
+  if (systemHead) result.system = encoding.decodeBootRecord(systemHead)
+  if (encryptionKeyBuffer) {
+    result.encryptionKey = encryptionKeyBuffer
+  }
+
+  if (!result.encryptionKey && (encryptionKey || encrypt)) {
+    if (!encryptionKey) {
+      encryptionKey = (await corestore.createKeyPair('autobee/encryption')).secretKey.subarray(
+        0,
+        32
+      )
+    }
+
+    await result.local.setUserData('autobee/encryption', encryptionKey)
+    result.encryptionKey = encryptionKey
+  }
+
+  return result
+}

package/lib/constants.js

@@ -0,0 +1,9 @@
+const LEGACY_OPLOG_VERSION = 2
+const OPLOG_VERSION = 3
+const AUTOBEE_VERSION = 3
+
+module.exports = {
+  LEGACY_OPLOG_VERSION,
+  OPLOG_VERSION,
+  AUTOBEE_VERSION
+}

package/lib/encoding.js

@@ -0,0 +1,150 @@
+const b4a = require('b4a')
+const c = require('compact-encoding')
+const crypto = require('hypercore-crypto')
+const { AutobeeEncryption } = require('autobee-encryption')
+
+const { getEncoding } = require('../spec/hyperschema')
+const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants')
+
+const Oplog = getEncoding('@autobee/oplog')
+const Link = getEncoding('@autobee/link')
+const SystemInfo = getEncoding('@autobee/system-info')
+const SystemWriter = getEncoding('@autobee/system-writer')
+const ManifestData = getEncoding('@autobee/manifest-data')
+
+exports.OPLOG_VERSION = OPLOG_VERSION
+
+exports.Oplog = Oplog
+exports.ManifestData = ManifestData
+exports.encodeBootRecord = encodeBootRecord
+exports.decodeBootRecord = decodeBootRecord
+exports.encodeSystemWriter = encodeSystemWriter
+exports.decodeSystemWriter = decodeSystemWriter
+
+function encodeBootRecord(r) {
+  return c.encode(Link, r)
+}
+
+function decodeBootRecord(value) {
+  return c.decode(Link, value)
+}
+
+function encodeSystemWriter(m) {
+  return c.encode(SystemWriter, m)
+}
+
+function decodeSystemWriter(key, value) {
+  return {
+    // TODO: add dummy type to schema to preset key without extra alloc
+    key: key.subarray(1),
+    ...c.decode(SystemWriter, value)
+  }
+}
+
+exports.encodeSystemInfo = encodeSystemInfo
+exports.decodeSystemInfo = decodeSystemInfo
+
+function encodeSystemInfo(m) {
+  return c.encode(SystemInfo, m)
+}
+
+function decodeSystemInfo(buf) {
+  return c.decode(SystemInfo, buf)
+}
+
+exports.encodeOplog = encodeOplog
+exports.decodeOplog = decodeOplog
+
+function decodeOplog(buf) {
+  const m = c.decode(Oplog, buf)
+
+  if (m.version <= LEGACY_OPLOG_VERSION) {
+    return {
+      version: m.version,
+      timestamp: 0,
+      links: m.node.heads,
+      batch: { start: 0, end: 0 },
+      views: null,
+      optimistic: !!m.optimistic,
+      value: m.node.value
+    }
+  }
+
+  if (m.batch === null) m.batch = { start: 0, end: 0 }
+  return m
+}
+
+function encodeOplog(m) {
+  if (m.batch && m.batch.start === 0 && m.batch.end === 0) m.batch = null
+  return c.encode(Oplog, m)
+}
+
+exports.encodeValue = encodeValue
+exports.decodeValue = decodeValue
+
+function createValue(value, opts = {}) {
+  if (opts.legacy) {
+    return {
+      version: opts.version || LEGACY_OPLOG_VERSION,
+      digest: null,
+      checkpoint: null,
+      optimistic: !!opts.optimistic,
+      node: {
+        heads: opts.heads || [],
+        batch: 1,
+        value
+      }
+    }
+  }
+
+  const {
+    version = OPLOG_VERSION,
+    optimistic = false,
+    timestamp = Date.now(),
+    links = [],
+    views = null
+  } = opts
+
+  return {
+    version,
+    timestamp,
+    links,
+    batch: null,
+    views,
+    optimistic,
+    value
+  }
+}
+
+function encodeValue(value, opts = {}) {
+  const oplog = createValue(value, opts)
+
+  const state = { start: 0, end: 0, buffer: null }
+
+  Oplog.preencode(state, oplog)
+
+  if (opts.padding) {
+    state.start = opts.padding
+    state.end += opts.padding
+  }
+
+  state.buffer = b4a.alloc(state.end)
+
+  Oplog.encode(state, oplog)
+
+  if (!opts.encrypted) return state.buffer
+
+  if (!opts.optimistic) {
+    throw new Error('Encoding an encrypted value is not supported')
+  }
+
+  const padding = b4a.alloc(16) // sodium.crypto_generichash_MINBYTES
+  crypto.hash(state.buffer, padding)
+  padding[0] = 0
+
+  return b4a.concat([padding.subarray(0, AutobeeEncryption.PADDING), state.buffer])
+}
+
+function decodeValue(buf, opts = {}) {
+  return decodeOplog(buf).value
+}

package/lib/system.js

@@ -0,0 +1,314 @@
+const Hyperbee = require('hyperbee2')
+const b4a = require('b4a')
+
+const { AUTOBEE_VERSION } = require('./constants')
+const encoding = require('./encoding.js')
+const topo = require('./topo.js')
+
+const INFO_KEY = b4a.from([0])
+const INFO_LEGACY_KEY = b4a.concat([b4a.from([0]), b4a.from('info')])
+const INDEXER_GTE = b4a.from([2])
+const INDEXER_LT = b4a.from([3])
+const EMPTY_HEAD = { length: 0, key: null }
+const EMPTY = b4a.from([])
+
+module.exports = class Systembee {
+  constructor(store, name, opts = {}) {
+    this.name = name // for debuggin
+    this.store = store
+    this.bee = new Hyperbee(store, opts)
+    this.view = null
+    this.heads = []
+    this.version = 0
+    this.timestamp = 0
+    this.flushes = 0
+    this.indexers = null // legacy
+    this.updates = new Map()
+    this.writers = new Map()
+    this.encrypted = opts.encrypted === true
+  }
+
+  async addWriter(key, { length = 0, isIndexer = false } = {}) {
+    if (length === 0) {
+      const info = await this.get(key, { unflushed: false })
+      length = info ? info.length : 0
+    }
+    this.update(key, length, isIndexer, false, true, false)
+  }
+
+  async ackWriter(key, { length = 0 } = {}) {
+    if (length === 0) {
+      const info = await this.get(key, { unflushed: false })
+      length = info ? info.length : 0
+    }
+    this.update(key, length, false, false, false, true)
+  }
+
+  async removeWriter(key, { length = 0 } = {}) {
+    if (length === 0) {
+      const info = await this.get(key, { unflushed: true })
+      length = info ? info.length : 0
+    }
+    this.update(key, length, false, true, false, false)
+  }
+
+  isGenesis() {
+    return this.bee.head() === null || this.bee.head().length === 0
+  }
+
+  async boot(view) {
+    await this.bee.ready()
+    this.bee.move(view)
+    await this.reset()
+  }
+
+  async getInfo() {
+    const node = await this.bee.get(INFO_KEY)
+    if (node) return encoding.decodeSystemInfo(node.value)
+    const legacy = await this.bee.get(INFO_LEGACY_KEY)
+    if (legacy) return encoding.decodeSystemInfo(legacy.value)
+    return null
+  }
+
+  async *getIndexers() {
+    for await (const data of this.bee.createReadStream({ gte: INDEXER_GTE, lt: INDEXER_LT })) {
+      const key = data.key.subarray(1)
+      yield key
+    }
+  }
+
+  async reset() {
+    const info = await this.getInfo()
+
+    this.version = info ? info.version : 0
+    this.view = info ? info.view : EMPTY_HEAD
+    this.heads = info ? info.heads : []
+    this.timestamp = info ? info.timestamp : 0
+    this.flushes = info ? info.flushes : 0
+    this.indexers = info ? info.indexers : null
+    this.updates.clear()
+    this.writers.clear()
+
+    if (this.version > AUTOBEE_VERSION) {
+      throw new Error('Autobee signals newer version than locally supported')
+    }
+  }
+
+  bootRecord() {
+    const system = this.bee.head()
+    return system.length ? system : null
+  }
+
+  async close() {
+    await this.bee.close()
+    await this.store.close()
+  }
+
+  async undo(head) {
+    this.bee.move(head)
+    await this.reset()
+    return this.view || EMPTY_HEAD
+  }
+
+  getLinks(key) {
+    const links = []
+    for (const h of this.heads) {
+      if (key && b4a.equals(h.key, key)) continue
+      links.push(h)
+    }
+    return links
+  }
+
+  addNode(node) {
+    for (let i = 0; i < this.heads.length; i++) {
+      const h = this.heads[i]
+      if (topo.isLinking(node, h)) {
+        this.heads.splice(i--, 1)
+      }
+    }
+
+    // Can enable if needed during debuggin
+    // asserts.heads(heads, node)
+
+    this.heads.push({ key: node.key, length: node.length })
+    if (node.timestamp > this.timestamp) this.timestamp = node.timestamp // TODO: support smoothing
+
+    this.update(node.key, node.length, false, false, false, false)
+  }
+
+  async canApply(key, optimistic) {
+    const id = b4a.toString(key, 'hex')
+    const w = this.writers.get(id)
+    if (w) return w.added
+    const info = await this.get(key)
+    return info ? !info.isRemoved : !!optimistic
+  }
+
+  async get(key, { unflushed = false } = {}) {
+    const node = await this.bee.get(encodeWriterKey(key))
+    const info = node !== null ? encoding.decodeSystemWriter(node.key, node.value) : null
+    if (!unflushed) return info
+
+    const upd = this.updates.get(b4a.toString(key, 'hex'))
+    if (!upd) return info
+
+    if (upd.isAdded) info.isAdded = true
+    if (upd.isRemoved) info.isRemoved = true
+    if (upd.isIndexer) info.isIndexer = true
+    if (upd.isAcked) info.isAcked = true
+    if (upd.isOplog) info.isOplog = true
+
+    info.length = upd.length
+
+    return info
+  }
+
+  async *list() {
+    for await (const data of this.bee.createReadStream()) {
+      if (data.key[0] === 1) {
+        yield encoding.decodeSystemWriter(data.key, data.value)
+      }
+    }
+  }
+
+  async has(link) {
+    // fast path
+    for (let i = 0; i < this.heads.length; i++) {
+      const h = this.heads[i]
+      if (b4a.equals(h.key, link.key)) return h.length >= link.length
+    }
+
+    const node = await this.get(link.key)
+    if (!node || node.length < link.length) return false
+    return true
+  }
+
+  update(key, length, isIndexer, isRemoved, isAdded, isAcked) {
+    const id = b4a.toString(key, 'hex')
+
+    let upd = this.updates.get(id)
+
+    if (isAdded) {
+      this.writers.set(id, { key, added: true })
+    } else if (isRemoved) {
+      this.writers.set(id, { key, added: false })
+    }
+
+    if (!upd) {
+      upd = { key, length, isIndexer, isRemoved, isAdded, isAcked, isOplog: false }
+      this.updates.set(id, upd)
+      return
+    }
+
+    if (isAdded) {
+      upd.isIndexer = isIndexer
+      upd.isRemoved = false
+    }
+    if (isRemoved) {
+      upd.isIndexer = false
+      upd.isRemoved = true
+    }
+    if (isAcked) {
+      upd.isAcked = true
+    }
+    if (length > upd.length) {
+      upd.length = length
+    }
+  }
+
+  async _updateWriter(upd, oplog) {
+    const changed = upd.isAdded || upd.isRemoved
+    const k = encodeWriterKey(upd.key)
+
+    if (upd.isAcked || !changed) {
+      const node = await this.bee.get(k)
+      const v = node ? encoding.decodeSystemWriter(node.key, node.value) : null
+
+      if (v) {
+        if (!changed) {
+          upd.isIndexer = v.isIndexer
+          upd.isRemoved = v.isRemoved
+        }
+
+        if (v.length > upd.length) upd.length = v.length
+      }
+    }
+
+    // TODO: we can optimise this a bit, if the above node hasnt changed, do not set this
+    // less writes, but future thing
+    upd.isOplog = b4a.equals(upd.key, oplog)
+
+    return [
+      k,
+      encoding.encodeSystemWriter(upd),
+      upd.isIndexer && upd.isAdded ? encodeIndexerWriterKey(upd.key) : null,
+      upd.isRemoved ? encodeIndexerWriterKey(upd.key) : null
+    ]
+  }
+
+  async flush(batch, bee) {
+    if (batch.length === 0) return []
+
+    const oplog = batch[batch.length - 1].key
+
+    const promises = []
+    for (const upd of this.updates.values()) {
+      promises.push(this._updateWriter(upd, oplog))
+    }
+
+    const w = this.bee.write()
+
+    // migrate
+    if (this.indexers) {
+      for (const key of this.indexers) w.tryPut(encodeIndexerWriterKey(key), EMPTY)
+      this.indexers = null
+    }
+
+    for (const [key, value, add, remove] of await Promise.all(promises)) {
+      w.tryPut(key, value)
+      if (add) w.tryPut(add, EMPTY)
+      if (remove) w.tryDelete(remove)
+    }
+
+    this.flushes++
+
+    const info = {
+      version: AUTOBEE_VERSION,
+      view: bee.head(),
+      heads: this.heads,
+      timestamp: this.timestamp,
+      flushes: this.flushes,
+      indexers: null // legacy
+    }
+
+    w.tryPut(INFO_KEY, encoding.encodeSystemInfo(info))
+
+    await w.flush()
+
+    // Can enable if needed during debuggin
+    // asserts.systemFlush(w)
+
+    this.updates.clear()
+
+    if (this.writers.size === 0) return []
+
+    const changes = [...this.writers.values()]
+
+    this.writers.clear()
+    return changes
+  }
+}
+
+function encodeWriterKey(key) {
+  const buf = b4a.allocUnsafe(33)
+  buf[0] = 1
+  buf.set(key, 1)
+  return buf
+}
+
+function encodeIndexerWriterKey(key) {
+  const buf = b4a.allocUnsafe(33)
+  buf[0] = 2
+  buf.set(key, 1)
+  return buf
+}