autobee 1.0.0 → 1.0.3

package/index.js

@@ -18,6 +18,7 @@ const { ActiveWriters } = require('./lib/writers.js')
 const UpdateChanges = require('./lib/updates.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
+const INTERRUPT = new Error('Apply interrupted')
 
 module.exports = class Autobee extends ReadyResource {
   constructor(store, key = null, handlers = {}) {
@@ -81,6 +82,10 @@ module.exports = class Autobee extends ReadyResource {
     this._updateLocalCore = null
     this._host = new ApplyCalls(this)
 
+    this.interrupted = null
+    this._interrupting = false
+    this._onErrorBound = this._onError.bind(this)
+
     this._wakeup = new AutobeeWakeup(this, handlers)
     this.wakeupCapability = null
 
@@ -126,6 +131,22 @@ module.exports = class Autobee extends ReadyResource {
     this._wakeup.setCapability(this.wakeupCapability.key, this.wakeupCapability.discoveryKey)
   }
 
+  getExternalWriters() {
+    const keys = []
+    for (const w of this.writers.active.values()) {
+      if (w === this.writers.localWriter) continue
+      keys.push(w.core.key)
+    }
+    return keys
+  }
+
+  async getWriterViews(key) {
+    const id = b4a.toString(key, 'hex')
+    const w = this.writers.active.get(id)
+    if (!w) return []
+    return w.views()
+  }
+
   views() {
     const sys = this.system.bee.context.local
     const view = this._workingBee.context.local
@@ -138,7 +159,8 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _close() {
-    await this.interrupt()
+    this._interrupting = true
+    if (this._draining) await this._draining
 
     if (this._handlers.close) await this._handlers.close(this.view)
 
@@ -254,8 +276,7 @@ module.exports = class Autobee extends ReadyResource {
     this.bumping++
 
     if (!this._draining) {
-      this._draining = this._drain()
-      this._draining.catch(safetyCatch)
+      this._draining = this._drain().catch(this._onErrorBound)
     }
 
     return this._draining
@@ -265,11 +286,46 @@ module.exports = class Autobee extends ReadyResource {
     return this._bump()
   }
 
-  updated() {
+  async updated() {
+    if (this.opened === false) await this.ready()
     if (this._draining) return this._draining
     return Promise.resolve()
   }
 
+  interrupt(reason) {
+    asserts.assert(!!this._host.applying, 'Interrupt is only allowed in apply')
+    this._interrupting = true
+    if (reason) this.interrupted = reason
+    throw INTERRUPT
+  }
+
+  getLastError() {
+    return this._lastError
+  }
+
+  _onError(err) {
+    if (this.closing) return
+
+    this._lastError = err
+
+    if (err === INTERRUPT) {
+      this.emit('interrupt', this.interrupted)
+      this.emit('update')
+      return
+    }
+
+    this.close().catch(safetyCatch)
+
+    // if no one is listening we should crash! we cannot rely on the EE here
+    // as this is wrapped in a promise so instead of nextTick throw it
+    if (ReadyResource.listenerCount(this, 'error') === 0) {
+      crashSoon(err)
+      return
+    }
+
+    this.emit('error', err)
+  }
+
   async _drain() {
     if (this._updateLocalCore !== null) {
       await this._rotateLocalWriter(this._updateLocalCore)
@@ -376,11 +432,6 @@ module.exports = class Autobee extends ReadyResource {
     this.emit('rotate-local-writer')
   }
 
-  interrupt() {
-    this._interrupting = true
-    if (this._draining) return this._draining
-  }
-
   async createAnchor() {
     const node = this._host.applying[this._host.applying.length - 1]
 
@@ -424,7 +475,7 @@ module.exports = class Autobee extends ReadyResource {
       await core.append(block, { writable: true, maxLength: 1 })
     }
 
-    await this.system.addWriter(core.key)
+    await this.system.addWriter(core.key, { weight: 1 })
 
     const anchor = { key: core.key, length: core.length }
 
@@ -475,8 +526,6 @@ module.exports = class Autobee extends ReadyResource {
       this._workingBee.move(t.view)
     }
 
-    asserts.assert(batch === t.tip[0], 'Batch must be first part of tip')
-
     let failed = true
 
     try {
@@ -594,9 +643,7 @@ module.exports = class Autobee extends ReadyResource {
       const value = values[i]
       const buffer = typeof value === 'string' ? b4a.from(value) : value
       const lnk = i === 0 ? links : []
-      const b = { start: i, end: values.length - 1 - i }
-
-      const node = this.writers.appendLocal(buffer, t, b, lnk, optimistic)
+      const node = this.writers.appendLocal(buffer, t, null, lnk, optimistic)
       batch.push(node)
     }
 
@@ -607,6 +654,10 @@ module.exports = class Autobee extends ReadyResource {
     // analyze is worth the trade off adding the view here also (technically not needed)
     await this.writers.flushLocal(this._workingBee.head())
   }
+
+  async replay() {
+    return topo.replay(this)
+  }
 }
 
 function isObject(o) {
@@ -634,3 +685,10 @@ function createAnchorCore(store, prologue, manifestData) {
 
   return core
 }
+
+function crashSoon(err) {
+  queueMicrotask(() => {
+    throw err
+  })
+  throw err
+}

package/lib/apply-calls.js

@@ -22,13 +22,17 @@ class ApplyCalls {
     return this.auto.name
   }
 
+  get clock() {
+    return this.auto.system.flushes
+  }
+
   get genesis() {
     return this.auto.system.isGenesis()
   }
 
-  addWriter(key, { isIndexer = true } = {}) {
+  addWriter(key, { isIndexer = true, weight = isIndexer ? 2 : 1 } = {}) {
     if (typeof key === 'string') key = ID.decode(key)
-    return this.auto.system.addWriter(key, { isIndexer })
+    return this.auto.system.addWriter(key, { weight })
   }
 
   ackWriter(key) {
@@ -46,8 +50,7 @@ class ApplyCalls {
   }
 
   interrupt(reason) {
-    throw new Error('TODO')
-    // this.auto._interrupt(reason)
+    this.auto.interrupt(reason)
   }
 
   removeable(key) {

package/lib/encoding.js

@@ -3,7 +3,7 @@ const c = require('compact-encoding')
 const crypto = require('hypercore-crypto')
 const { AutobeeEncryption } = require('autobee-encryption')
 
-const { getEncoding } = require('../spec/hyperschema')
+const { getEncoding } = require('../encoding/spec/autobee')
 const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants')
 
 const Oplog = getEncoding('@autobee/oplog')

package/lib/system.js

@@ -28,12 +28,12 @@ module.exports = class Systembee {
     this.encrypted = opts.encrypted === true
   }
 
-  async addWriter(key, { length = 0, isIndexer = false } = {}) {
+  async addWriter(key, { length = 0, weight = 1 } = {}) {
     if (length === 0) {
       const info = await this.get(key, { unflushed: false })
       length = info ? info.length : 0
     }
-    this.update(key, length, isIndexer, false, true, false)
+    this.update(key, length, weight, false, true, false)
   }
 
   async ackWriter(key, { length = 0 } = {}) {
@@ -41,7 +41,7 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: false })
       length = info ? info.length : 0
     }
-    this.update(key, length, false, false, false, true)
+    this.update(key, length, 0, false, false, true)
   }
 
   async removeWriter(key, { length = 0 } = {}) {
@@ -49,7 +49,7 @@ module.exports = class Systembee {
       const info = await this.get(key, { unflushed: true })
       length = info ? info.length : 0
     }
-    this.update(key, length, false, true, false, false)
+    this.update(key, length, 0, true, false, false)
   }
 
   isGenesis() {
@@ -133,7 +133,7 @@ module.exports = class Systembee {
     this.heads.push({ key: node.key, length: node.length })
     if (node.timestamp > this.timestamp) this.timestamp = node.timestamp // TODO: support smoothing
 
-    this.update(node.key, node.length, false, false, false, false)
+    this.update(node.key, node.length, 0, false, false, false)
   }
 
   async canApply(key, optimistic) {
@@ -151,6 +151,7 @@ module.exports = class Systembee {
 
     const upd = this.updates.get(b4a.toString(key, 'hex'))
     if (!upd) return info
+    if (!info) return upd
 
     if (upd.isAdded) info.isAdded = true
     if (upd.isRemoved) info.isRemoved = true
@@ -183,7 +184,7 @@ module.exports = class Systembee {
     return true
   }
 
-  update(key, length, isIndexer, isRemoved, isAdded, isAcked) {
+  update(key, length, weight, isRemoved, isAdded, isAcked) {
     const id = b4a.toString(key, 'hex')
 
     let upd = this.updates.get(id)
@@ -195,17 +196,27 @@ module.exports = class Systembee {
     }
 
     if (!upd) {
-      upd = { key, length, isIndexer, isRemoved, isAdded, isAcked, isOplog: false }
+      upd = {
+        version: 4,
+        key,
+        length,
+        weight,
+        clock: 0,
+        isRemoved,
+        isAdded,
+        isAcked,
+        isOplog: false
+      }
       this.updates.set(id, upd)
       return
     }
 
     if (isAdded) {
-      upd.isIndexer = isIndexer
+      upd.weight = weight
       upd.isRemoved = false
     }
     if (isRemoved) {
-      upd.isIndexer = false
+      upd.weight = 0
       upd.isRemoved = true
     }
     if (isAcked) {
@@ -226,7 +237,7 @@ module.exports = class Systembee {
 
       if (v) {
         if (!changed) {
-          upd.isIndexer = v.isIndexer
+          upd.weight = v.weight
           upd.isRemoved = v.isRemoved
         }
 
@@ -234,6 +245,8 @@ module.exports = class Systembee {
       }
     }
 
+    upd.clock = this.flushes
+
     // TODO: we can optimise this a bit, if the above node hasnt changed, do not set this
     // less writes, but future thing
     upd.isOplog = b4a.equals(upd.key, oplog)

package/lib/topo.js

@@ -5,11 +5,43 @@ const asserts = require('./asserts.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 
+exports.replay = replay
 exports.sort = sort
 
 exports.isLinking = isLinking
 exports.isLinkingAll = isLinkingAll
 
+class Clock {
+  constructor() {
+    this.map = new Map()
+  }
+
+  add(link) {
+    const hex = b4a.toString(link.key, 'hex')
+    const p = this.map.get(hex)
+    if (p < link.length) this.map.set(hex, link.length)
+  }
+
+  has(link) {
+    const hex = b4a.toString(link.key, 'hex')
+    const p = this.map.get(hex)
+    return p >= link.length
+  }
+}
+
+async function replay(auto) {
+  const ch = auto.system.bee.createChangesStream()
+  const replay = []
+
+  for await (const data of ch) {
+    const { ref, info } = getOplog(data)
+    replay.push(ref)
+  }
+
+  const inflated = await inflate(auto, replay.reverse())
+  return inflated.flatMap((b) => b.batch)
+}
+
 async function sort(auto, batch) {
   const node = batch[0]
   const ch = auto.system.bee.createChangesStream()
@@ -17,52 +49,98 @@ async function sort(auto, batch) {
 
   for await (const data of ch) {
     const { ref, info } = getOplog(data)
-    const link = { key: ref.oplog.key, length: ref.oplog.length }
+    const link = {
+      key: ref.oplog.key,
+      length: ref.oplog.length,
+      weight: ref.oplog.weight,
+      timestamp: info.timestamp
+    }
 
-    if (isLinking(node, link) || node.timestamp > info.timestamp) break
+    if (!isOrderedBefore(node, link)) break
     reorder.push(ref)
   }
 
   const inflated = await inflate(auto, reorder.reverse())
 
-  for (let i = 0; i < inflated.length; i++) {
-    const b = inflated[i]
-    const op = b.batch[0] // just use first op, they are all the same writer
-    const c = cmp(op, node)
+  const { undo, shared } = addSorted(inflated, batch)
+
+  const tip = inflated.slice(shared).map((b) => b.batch)
 
-    if (c === -1) continue
+  return {
+    undo,
+    view: null,
+    tip
+  }
+}
+
+function isOrderedBefore(node, oplog) {
+  const { version, weight, timestamp } = oplog
+  if (version === 0) return false
+  if (node.weight < weight) return false
+  if (isLinking(node, oplog)) return false
+  return node.timestamp <= timestamp
+}
 
-    const tip = new Array(inflated.length - i + 1)
-    tip[0] = batch
-    for (let j = i; j < inflated.length; j++) {
-      tip[j - i + 1] = inflated[j].batch
+function addSorted(list, batch) {
+  const node = batch[0]
+  const clock = new Clock()
+  for (const link of node.links) {
+    clock.add(link)
+  }
+
+  const tip = []
+  const linked = []
+
+  let undo = null
+  let shared = list.length
+
+  while (list.length) {
+    // reached a stable sorting point
+    if (cmp(node, list[list.length - 1].batch[0]) > 0) {
+      break
     }
 
-    return {
-      undo: b.undo,
-      view: null,
-      tip
+    const b = list.pop()
+    const target = b.batch[0]
+
+    if (list.length < shared) {
+      undo = b.undo
+      shared = list.length
     }
-  }
 
-  return {
-    undo: null,
-    view: null,
-    tip: [batch]
+    // move past this node
+    if (!clock.has(target)) {
+      tip.push(b)
+      continue
+    }
+
+    // remove node
+    linked.push(b)
+
+    // update the clock to catch linked nodes
+    for (const link of target.links) {
+      clock.add(link)
+    }
   }
+
+  while (linked.length) addSorted(list, linked.pop())
+  list.push({ undo: null, batch })
+  while (tip.length) list.push(tip.pop())
+
+  return { undo, shared }
 }
 
 function cmp(a, b) {
-  const c = b4a.compare(a.key, b.key)
-
-  if (c === 0) {
-    return a.length - b.length
-  }
+  const w = a.weight - b.weight
+  if (w) return w > 0 ? -1 : 1
 
   const t = a.timestamp - b.timestamp
+  if (t) return t < 0 ? -1 : 1
 
-  if (t === 0) return c
-  return t < 0 ? -1 : 1
+  const c = b4a.compare(a.key, b.key)
+  if (c) return c
+
+  return a.length < b.length ? -1 : 1
 }
 
 async function inflate(auto, reorder) {
@@ -74,7 +152,7 @@ async function inflate(auto, reorder) {
 
     const local = auto.writers.localWriter
     if (local && local.id === id) {
-      promises.push(getLocalBatch(ref.undo, local, ref.oplog.length))
+      promises.push(getLocalBatch(ref.undo, local, ref.oplog.length, ref.oplog.weight))
       continue
     }
 
@@ -84,7 +162,7 @@ async function inflate(auto, reorder) {
       sessions.set(id, core)
     }
 
-    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length))
+    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, ref.oplog.weight))
   }
 
   try {
@@ -94,11 +172,11 @@ async function inflate(auto, reorder) {
   }
 }
 
-async function getOplogBatch(undo, core, length) {
+async function getOplogBatch(undo, core, length, weight) {
   const seq = length - 1
   const block = await core.get(seq)
   const oplog = encoding.decodeOplog(block)
-  const head = createNode(core, length, oplog)
+  const head = createNode(core, length, weight, oplog)
 
   const batch = []
   const result = { undo, batch }
@@ -115,7 +193,7 @@ async function getOplogBatch(undo, core, length) {
     const blocks = await Promise.all(remaining)
     for (let i = 0; i < blocks.length; i++) {
       const oplog = encoding.decodeOplog(blocks[i])
-      batch.push(createNode(core, i + start + 1, oplog))
+      batch.push(createNode(core, i + start + 1, weight, oplog))
     }
   }
 
@@ -123,7 +201,7 @@ async function getOplogBatch(undo, core, length) {
   return result
 }
 
-async function getLocalBatch(undo, writer, length) {
+function getLocalBatch(undo, writer, length, weight) {
   if (length > writer.core.length) {
     if (!writer.pending) throw new Error('No local nodes')
 
@@ -142,16 +220,13 @@ async function getLocalBatch(undo, writer, length) {
     return { undo, batch }
   }
 
-  return getOplogBatch(undo, writer.core, length)
+  return getOplogBatch(undo, writer.core, length, weight)
 }
 
 function getOplog(data) {
   const result = { ref: null, info: null }
-  for (let i = 0; i < data.batch.length; i++) {
-    const keys = data.batch[i].keys
-
-    for (let j = 0; j < keys.length; j++) {
-      const k = keys[j]
+  for (const { keys } of data.batch) {
+    for (const k of keys) {
       const prefix = k.key[0]
 
       if (prefix === 0) {