autobee 1.0.0 → 1.0.2

package/README.md

@@ -2,13 +2,253 @@
 
 Unstoppable, scalable multiwriter Hyperbee.
 
+> **Still experimental and under heavy development. Expect breaking changes.**
+
 ```sh
-npm i autobee
+npm install autobee
+```
+
+Multiple peers each write to their own local Hypercore. An `apply` function you provide merges those writes into a shared Hyperbee view deterministically. The view is consistent across all peers once they replicate.
+
+## Usage
+
+```js
+const Autobee = require('autobee')
+const Corestore = require('corestore')
+
+const store = new Corestore('./my-db')
+
+const db = new Autobee(store, null, { apply })
+await db.ready()
+
+// append some data
+await db.append(Buffer.from(JSON.stringify({ hello: 'world' })))
+
+// read it back from the view
+const node = await db.view.get(Buffer.from('latest'))
+console.log(JSON.parse(node.value))
+
+async function apply(nodes, view, host) {
+  for (const node of nodes) {
+    const op = JSON.parse(node.value)
+
+    if (op.addWriter) host.addWriter(op.addWriter)
+    if (op.removeWriter) host.removeWriter(op.removeWriter)
+
+    const w = view.write()
+    w.tryPut(Buffer.from('latest'), node.value)
+    await w.flush()
+  }
+}
+```
+
+To add a second writer and replicate:
+
+```js
+const db1 = new Autobee(store1, null, { apply })
+await db1.ready()
+
+// share db1.key with others so they can join
+const db2 = new Autobee(store2, db1.key, { apply })
+await db2.ready()
+
+// db1 adds db2 as a writer
+await db1.append(Buffer.from(JSON.stringify({ addWriter: db2.local.id })))
+
+// replicate using any stream
+const s1 = db1.replicate(true)
+const s2 = db2.replicate(false)
+s1.pipe(s2).pipe(s1)
 ```
 
 ## API
 
-#### `const auto = new Autobee(corestore, [key], [options])`
+#### `const db = new Autobee(store, [key], [options])`
+
+Create a new Autobee. `store` is a Corestore. `key` is the public key of an existing Autobee to join — omit or pass `null` to create a new one.
+
+Options:
+
+```js
+{
+  apply (nodes, view, host) {},  // called with batches of new nodes to apply to the view
+  open (bee, db) {},             // called to create a custom view, return it
+  close (view) {},               // called when the db closes
+  update (view, changes) {},     // called after apply when the view has been updated
+  encryptionKey: Buffer,         // 32-byte key to encrypt all data at rest
+  encrypted: false,              // set true if using encryptionKey
+  keyPair: { publicKey, secretKey }, // custom signing key pair for the local writer
+  optimistic: true               // allow optimistic writes from unknown writers
+}
+```
+
+#### `db.key`
+
+The public key of this Autobee. Share this with peers so they can join.
+
+#### `db.discoveryKey`
+
+The discovery key. Use this to find peers on the network.
+
+#### `db.id`
+
+The public key encoded as a hex string.
+
+#### `db.local`
+
+The local writer Hypercore. Use `db.local.key` or `db.local.id` to identify this writer to others.
+
+#### `db.view`
+
+A read-only snapshot of the Hyperbee view. Updated after each apply cycle. Use the standard [Hyperbee](https://github.com/holepunks/hyperbee) API to read from it.
+
+#### `db.bee`
+
+Alias for `db.view`.
+
+#### `db.writable`
+
+`true` if this instance has been added as a writer.
+
+#### `db.isIndexer`
+
+`true` if this writer is an indexer.
+
+#### `await db.append(value | values)`
+
+Append one or more values to the local writer. Triggers an apply cycle.
+
+```js
+await db.append(Buffer.from('hello'))
+await db.append([buf1, buf2, buf3])
+```
+
+Optionally pass `{ optimistic: true }` to write without waiting to be a confirmed writer.
+
+```js
+await db.append(buf, { optimistic: true })
+```
+
+#### `await db.update()`
+
+Trigger a new apply cycle. Useful after replication to process new data.
+
+#### `await db.updated()`
+
+Wait until the current apply cycle has finished.
+
+#### `await db.flush()`
+
+Wait until all known writers have been fully indexed.
+
+#### `stream = db.replicate(isInitiator)`
+
+Create a replication stream. Pass `true` for the initiating side, `false` for the other.
+
+```js
+const s1 = db1.replicate(true)
+const s2 = db2.replicate(false)
+s1.pipe(s2).pipe(s1)
+```
+
+#### `db.wakeup({ key, length })`
+
+Hint that a new writer core is available at `key` with at least `length` entries. Used to wake up replication when you learn about a peer out of band.
+
+#### `await db.setLocal(key, [options])`
+
+Rotate the local writer to a different key. The new writer takes over as the active oplog.
+
+#### `views = db.views()`
+
+Returns the current system and view core positions. Used for replication coordination.
+
+#### `Autobee.isAutobee(val)`
+
+Returns `true` if `val` is an Autobee instance.
+
+### Apply
+
+The `apply` function is called with a batch of nodes from writers, a writable `view` (Hyperbee batch), and a `host` object.
+
+```js
+async function apply(nodes, view, host) {
+  for (const node of nodes) {
+    // node.key    — writer public key (Buffer)
+    // node.value  — the value appended (Buffer)
+    // node.length — position in the writer's core
+
+    const op = JSON.parse(node.value)
+
+    // manage writers
+    if (op.addWriter) host.addWriter(op.addWriter)
+    if (op.removeWriter) host.removeWriter(op.removeWriter)
+
+    // write to the view
+    const w = view.write()
+    w.tryPut(Buffer.from('key'), node.value)
+    await w.flush()
+  }
+}
+```
+
+#### `host.addWriter(key, [options])`
+
+Add a writer by public key (Buffer or hex string). Options:
+
+```js
+{
+  isIndexer: true // default
+}
+```
+
+#### `host.removeWriter(key)`
+
+Remove a writer by public key (Buffer or hex string).
+
+#### `host.ackWriter(key)`
+
+Acknowledge a writer without changing their permissions.
+
+#### `host.interrupt(reason)`
+
+Interrupt the current apply cycle. The db emits `'interrupt'` with the reason. Useful for pausing apply while waiting on external data.
+
+#### `anchor = await host.createAnchor()`
+
+Create an anchor node. Returns `{ key, length }`. Anchors are used to create a verifiable checkpoint in the log that can be used by future writers to prove causal ordering.
+
+#### `host.genesis`
+
+`true` if the system has not yet processed any nodes. Use this to bootstrap the first writer.
+
+### Encryption
+
+Pass an `encryptionKey` to encrypt all writer cores and the view at rest.
+
+```js
+const db = new Autobee(store, null, {
+  apply,
+  encrypted: true,
+  encryptionKey: crypto.randomBytes(32)
+})
+```
+
+All peers must use the same encryption key.
+
+### Static methods
+
+#### `buf = Autobee.encodeValue(value, [opts])`
+
+Encode a value into an Autobee block with optional metadata.
+
+#### `value = Autobee.decodeValue(buf, [opts])`
+
+Decode an Autobee block back to its value.
+
+#### `Autobee.GENESIS`
+
+`{ length: 0, key: null }`. The empty head used to represent the genesis state.
 
 ## License
 

package/index.js

@@ -18,6 +18,7 @@ const { ActiveWriters } = require('./lib/writers.js')
 const UpdateChanges = require('./lib/updates.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
+const INTERRUPT = new Error('Apply interrupted')
 
 module.exports = class Autobee extends ReadyResource {
   constructor(store, key = null, handlers = {}) {
@@ -81,6 +82,10 @@ module.exports = class Autobee extends ReadyResource {
     this._updateLocalCore = null
     this._host = new ApplyCalls(this)
 
+    this.interrupted = null
+    this._interrupting = false
+    this._onErrorBound = this._onError.bind(this)
+
     this._wakeup = new AutobeeWakeup(this, handlers)
     this.wakeupCapability = null
 
@@ -126,6 +131,22 @@ module.exports = class Autobee extends ReadyResource {
     this._wakeup.setCapability(this.wakeupCapability.key, this.wakeupCapability.discoveryKey)
   }
 
+  getExternalWriters() {
+    const keys = []
+    for (const w of this.writers.active.values()) {
+      if (w === this.writers.localWriter) continue
+      keys.push(w.core.key)
+    }
+    return keys
+  }
+
+  async getWriterViews(key) {
+    const id = b4a.toString(key, 'hex')
+    const w = this.writers.active.get(id)
+    if (!w) return []
+    return w.views()
+  }
+
   views() {
     const sys = this.system.bee.context.local
     const view = this._workingBee.context.local
@@ -138,7 +159,8 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _close() {
-    await this.interrupt()
+    this._interrupting = true
+    if (this._draining) await this._draining
 
     if (this._handlers.close) await this._handlers.close(this.view)
 
@@ -254,8 +276,7 @@ module.exports = class Autobee extends ReadyResource {
     this.bumping++
 
     if (!this._draining) {
-      this._draining = this._drain()
-      this._draining.catch(safetyCatch)
+      this._draining = this._drain().catch(this._onErrorBound)
     }
 
     return this._draining
@@ -265,11 +286,46 @@ module.exports = class Autobee extends ReadyResource {
     return this._bump()
   }
 
-  updated() {
+  async updated() {
+    if (this.opened === false) await this.ready()
     if (this._draining) return this._draining
     return Promise.resolve()
   }
 
+  interrupt(reason) {
+    asserts.assert(!!this._host.applying, 'Interrupt is only allowed in apply')
+    this._interrupting = true
+    if (reason) this.interrupted = reason
+    throw INTERRUPT
+  }
+
+  getLastError() {
+    return this._lastError
+  }
+
+  _onError(err) {
+    if (this.closing) return
+
+    this._lastError = err
+
+    if (err === INTERRUPT) {
+      this.emit('interrupt', this.interrupted)
+      this.emit('update')
+      return
+    }
+
+    this.close().catch(safetyCatch)
+
+    // if no one is listening we should crash! we cannot rely on the EE here
+    // as this is wrapped in a promise so instead of nextTick throw it
+    if (ReadyResource.listenerCount(this, 'error') === 0) {
+      crashSoon(err)
+      return
+    }
+
+    this.emit('error', err)
+  }
+
   async _drain() {
     if (this._updateLocalCore !== null) {
       await this._rotateLocalWriter(this._updateLocalCore)
@@ -376,11 +432,6 @@ module.exports = class Autobee extends ReadyResource {
     this.emit('rotate-local-writer')
   }
 
-  interrupt() {
-    this._interrupting = true
-    if (this._draining) return this._draining
-  }
-
   async createAnchor() {
     const node = this._host.applying[this._host.applying.length - 1]
 
@@ -424,7 +475,7 @@ module.exports = class Autobee extends ReadyResource {
       await core.append(block, { writable: true, maxLength: 1 })
     }
 
-    await this.system.addWriter(core.key)
+    await this.system.addWriter(core.key, { weight: 1 })
 
     const anchor = { key: core.key, length: core.length }
 
@@ -475,8 +526,6 @@ module.exports = class Autobee extends ReadyResource {
       this._workingBee.move(t.view)
     }
 
-    asserts.assert(batch === t.tip[0], 'Batch must be first part of tip')
-
     let failed = true
 
     try {
@@ -594,9 +643,7 @@ module.exports = class Autobee extends ReadyResource {
       const value = values[i]
       const buffer = typeof value === 'string' ? b4a.from(value) : value
       const lnk = i === 0 ? links : []
-      const b = { start: i, end: values.length - 1 - i }
-
-      const node = this.writers.appendLocal(buffer, t, b, lnk, optimistic)
+      const node = this.writers.appendLocal(buffer, t, null, lnk, optimistic)
       batch.push(node)
     }
 
@@ -607,6 +654,10 @@ module.exports = class Autobee extends ReadyResource {
     // analyze is worth the trade off adding the view here also (technically not needed)
     await this.writers.flushLocal(this._workingBee.head())
   }
+
+  async replay() {
+    return topo.replay(this)
+  }
 }
 
 function isObject(o) {
@@ -634,3 +685,10 @@ function createAnchorCore(store, prologue, manifestData) {
 
   return core
 }
+
+function crashSoon(err) {
+  queueMicrotask(() => {
+    throw err
+  })
+  throw err
+}

package/lib/apply-calls.js

@@ -22,13 +22,17 @@ class ApplyCalls {
     return this.auto.name
   }
 
+  get clock() {
+    return this.auto.system.flushes
+  }
+
   get genesis() {
     return this.auto.system.isGenesis()
   }
 
-  addWriter(key, { isIndexer = true } = {}) {
+  addWriter(key, { isIndexer = true, weight = isIndexer ? 2 : 1 } = {}) {
     if (typeof key === 'string') key = ID.decode(key)
-    return this.auto.system.addWriter(key, { isIndexer })
+    return this.auto.system.addWriter(key, { weight })
   }
 
   ackWriter(key) {
@@ -46,8 +50,7 @@ class ApplyCalls {
   }
 
   interrupt(reason) {
-    throw new Error('TODO')
-    // this.auto._interrupt(reason)
+    this.auto.interrupt(reason)
   }
 
   removeable(key) {

package/lib/encoding.js

@@ -3,7 +3,7 @@ const c = require('compact-encoding')
 const crypto = require('hypercore-crypto')
 const { AutobeeEncryption } = require('autobee-encryption')
 
-const { getEncoding } = require('../spec/hyperschema')
+const { getEncoding } = require('../encoding/spec/autobee')
 const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants')
 
 const Oplog = getEncoding('@autobee/oplog')

package/lib/system.js

@@ -28,12 +28,12 @@ module.exports = class Systembee {
     this.encrypted = opts.encrypted === true
   }
 
-  async addWriter(key, { length = 0, isIndexer = false } = {}) {
+  async addWriter(key, { length = 0, weight = 1 } = {}) {
     if (length === 0) {
       const info = await this.get(key, { unflushed: false })
       length = info ? info.length : 0
     }
-    this.update(key, length, isIndexer, false, true, false)
+    this.update(key, length, weight, false, true, false)
   }
 
   async ackWriter(key, { length = 0 } = {}) {
@@ -41,7 +41,7 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: false })
       length = info ? info.length : 0
     }
-    this.update(key, length, false, false, false, true)
+    this.update(key, length, 0, false, false, true)
   }
 
   async removeWriter(key, { length = 0 } = {}) {
@@ -49,7 +49,7 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: true })
       length = info ? info.length : 0
     }
-    this.update(key, length, false, true, false, false)
+    this.update(key, length, 0, true, false, false)
   }
 
   isGenesis() {
@@ -133,7 +133,7 @@ module.exports = class Systembee {
     this.heads.push({ key: node.key, length: node.length })
     if (node.timestamp > this.timestamp) this.timestamp = node.timestamp // TODO: support smoothing
 
-    this.update(node.key, node.length, false, false, false, false)
+    this.update(node.key, node.length, 0, false, false, false)
   }
 
   async canApply(key, optimistic) {
@@ -151,6 +151,7 @@ module.exports = class Systembee {
 
     const upd = this.updates.get(b4a.toString(key, 'hex'))
     if (!upd) return info
+    if (!info) return upd
 
     if (upd.isAdded) info.isAdded = true
     if (upd.isRemoved) info.isRemoved = true
@@ -183,7 +184,7 @@ module.exports = class Systembee {
     return true
   }
 
-  update(key, length, isIndexer, isRemoved, isAdded, isAcked) {
+  update(key, length, weight, isRemoved, isAdded, isAcked) {
     const id = b4a.toString(key, 'hex')
 
     let upd = this.updates.get(id)
@@ -195,17 +196,27 @@ module.exports = class Systembee {
     }
 
     if (!upd) {
-      upd = { key, length, isIndexer, isRemoved, isAdded, isAcked, isOplog: false }
+      upd = {
+        version: 4,
+        key,
+        length,
+        weight,
+        clock: 0,
+        isRemoved,
+        isAdded,
+        isAcked,
+        isOplog: false
+      }
       this.updates.set(id, upd)
       return
     }
 
     if (isAdded) {
-      upd.isIndexer = isIndexer
+      upd.weight = weight
       upd.isRemoved = false
     }
     if (isRemoved) {
-      upd.isIndexer = false
+      upd.weight = 0
       upd.isRemoved = true
     }
     if (isAcked) {
@@ -226,7 +237,7 @@ module.exports = class Systembee {
 
       if (v) {
         if (!changed) {
-          upd.isIndexer = v.isIndexer
+          upd.weight = v.weight
           upd.isRemoved = v.isRemoved
         }
 
@@ -234,6 +245,8 @@ module.exports = class Systembee {
       }
     }
 
+    upd.clock = this.flushes
+
     // TODO: we can optimise this a bit, if the above node hasnt changed, do not set this
     // less writes, but future thing
     upd.isOplog = b4a.equals(upd.key, oplog)