autobee 1.0.0 → 1.0.2

package/lib/topo.js

@@ -5,11 +5,43 @@ const asserts = require('./asserts.js')
 
 const EMPTY_HEAD = { length: 0, key: null }
 
+exports.replay = replay
 exports.sort = sort
 
 exports.isLinking = isLinking
 exports.isLinkingAll = isLinkingAll
 
+class Clock {
+  constructor() {
+    this.map = new Map()
+  }
+
+  add(link) {
+    const hex = b4a.toString(link.key, 'hex')
+    const p = this.map.get(hex)
+    if (p < link.length) this.map.set(hex, link.length)
+  }
+
+  has(link) {
+    const hex = b4a.toString(link.key, 'hex')
+    const p = this.map.get(hex)
+    return p >= link.length
+  }
+}
+
+async function replay(auto) {
+  const ch = auto.system.bee.createChangesStream()
+  const replay = []
+
+  for await (const data of ch) {
+    const { ref, info } = getOplog(data)
+    replay.push(ref)
+  }
+
+  const inflated = await inflate(auto, replay.reverse())
+  return inflated.flatMap((b) => b.batch)
+}
+
 async function sort(auto, batch) {
   const node = batch[0]
   const ch = auto.system.bee.createChangesStream()
@@ -17,52 +49,98 @@ async function sort(auto, batch) {
 
   for await (const data of ch) {
     const { ref, info } = getOplog(data)
-    const link = { key: ref.oplog.key, length: ref.oplog.length }
+    const link = {
+      key: ref.oplog.key,
+      length: ref.oplog.length,
+      weight: ref.oplog.weight,
+      timestamp: info.timestamp
+    }
 
-    if (isLinking(node, link) || node.timestamp > info.timestamp) break
+    if (!isOrderedBefore(node, link)) break
     reorder.push(ref)
   }
 
   const inflated = await inflate(auto, reorder.reverse())
 
-  for (let i = 0; i < inflated.length; i++) {
-    const b = inflated[i]
-    const op = b.batch[0] // just use first op, they are all the same writer
-    const c = cmp(op, node)
+  const { undo, shared } = addSorted(inflated, batch)
+
+  const tip = inflated.slice(shared).map((b) => b.batch)
 
-    if (c === -1) continue
+  return {
+    undo,
+    view: null,
+    tip
+  }
+}
+
+function isOrderedBefore(node, oplog) {
+  const { version, weight, timestamp } = oplog
+  if (version === 0) return false
+  if (node.weight < weight) return false
+  if (isLinking(node, oplog)) return false
+  return node.timestamp <= timestamp
+}
 
-    const tip = new Array(inflated.length - i + 1)
-    tip[0] = batch
-    for (let j = i; j < inflated.length; j++) {
-      tip[j - i + 1] = inflated[j].batch
+function addSorted(list, batch) {
+  const node = batch[0]
+  const clock = new Clock()
+  for (const link of node.links) {
+    clock.add(link)
+  }
+
+  const tip = []
+  const linked = []
+
+  let undo = null
+  let shared = list.length
+
+  while (list.length) {
+    // reached a stable sorting point
+    if (cmp(node, list[list.length - 1].batch[0]) > 0) {
+      break
     }
 
-    return {
-      undo: b.undo,
-      view: null,
-      tip
+    const b = list.pop()
+    const target = b.batch[0]
+
+    if (list.length < shared) {
+      undo = b.undo
+      shared = list.length
     }
-  }
 
-  return {
-    undo: null,
-    view: null,
-    tip: [batch]
+    // move past this node
+    if (!clock.has(target)) {
+      tip.push(b)
+      continue
+    }
+
+    // remove node
+    linked.push(b)
+
+    // update the clock to catch linked nodes
+    for (const link of target.links) {
+      clock.add(link)
+    }
   }
+
+  while (linked.length) addSorted(list, linked.pop())
+  list.push({ undo: null, batch })
+  while (tip.length) list.push(tip.pop())
+
+  return { undo, shared }
 }
 
 function cmp(a, b) {
-  const c = b4a.compare(a.key, b.key)
-
-  if (c === 0) {
-    return a.length - b.length
-  }
+  const w = a.weight - b.weight
+  if (w) return w > 0 ? -1 : 1
 
   const t = a.timestamp - b.timestamp
+  if (t) return t < 0 ? -1 : 1
 
-  if (t === 0) return c
-  return t < 0 ? -1 : 1
+  const c = b4a.compare(a.key, b.key)
+  if (c) return c
+
+  return a.length < b.length ? -1 : 1
 }
 
 async function inflate(auto, reorder) {
@@ -74,7 +152,7 @@ async function inflate(auto, reorder) {
 
     const local = auto.writers.localWriter
     if (local && local.id === id) {
-      promises.push(getLocalBatch(ref.undo, local, ref.oplog.length))
+      promises.push(getLocalBatch(ref.undo, local, ref.oplog.length, ref.oplog.weight))
       continue
     }
 
@@ -84,7 +162,7 @@ async function inflate(auto, reorder) {
       sessions.set(id, core)
     }
 
-    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length))
+    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, ref.oplog.weight))
   }
 
   try {
@@ -94,11 +172,11 @@ async function inflate(auto, reorder) {
   }
 }
 
-async function getOplogBatch(undo, core, length) {
+async function getOplogBatch(undo, core, length, weight) {
   const seq = length - 1
   const block = await core.get(seq)
   const oplog = encoding.decodeOplog(block)
-  const head = createNode(core, length, oplog)
+  const head = createNode(core, length, weight, oplog)
 
   const batch = []
   const result = { undo, batch }
@@ -115,7 +193,7 @@ async function getOplogBatch(undo, core, length) {
     const blocks = await Promise.all(remaining)
     for (let i = 0; i < blocks.length; i++) {
       const oplog = encoding.decodeOplog(blocks[i])
-      batch.push(createNode(core, i + start + 1, oplog))
+      batch.push(createNode(core, i + start + 1, weight, oplog))
     }
   }
 
@@ -123,7 +201,7 @@ async function getOplogBatch(undo, core, length) {
   return result
 }
 
-async function getLocalBatch(undo, writer, length) {
+function getLocalBatch(undo, writer, length, weight) {
   if (length > writer.core.length) {
     if (!writer.pending) throw new Error('No local nodes')
 
@@ -142,16 +220,13 @@ async function getLocalBatch(undo, writer, length) {
     return { undo, batch }
   }
 
-  return getOplogBatch(undo, writer.core, length)
+  return getOplogBatch(undo, writer.core, length, weight)
 }
 
 function getOplog(data) {
   const result = { ref: null, info: null }
-  for (let i = 0; i < data.batch.length; i++) {
-    const keys = data.batch[i].keys
-
-    for (let j = 0; j < keys.length; j++) {
-      const k = keys[j]
+  for (const { keys } of data.batch) {
+    for (const k of keys) {
       const prefix = k.key[0]
 
       if (prefix === 0) {

package/lib/writers.js

@@ -3,6 +3,8 @@ const encoding = require('./encoding.js')
 const Triggers = require('./triggers.js')
 const { WriterEncryption } = require('autobee-encryption')
 
+const WRITER_PREFETCH = 10
+
 const BATCH_WRITER_SANITY = 1024
 const BATCH_OPTIMISTIC_SANITY = 32
 
@@ -80,7 +82,7 @@ class ActiveWriters {
     const info = await this.auto.system.get(this.auto.local.key)
     const writable = this.writable
 
-    w.isIndexer = info ? info.isIndexer : false
+    w.weight = info ? info.weight : 0
     this.writable = info ? !info.isRemoved : bootstrapping
 
     if (writable === this.writable) return
@@ -158,23 +160,30 @@ class Writer {
     this.core = core
     this.id = id
     this.index = 0
+    this.weight = 0
     this.pending = null
     this.waiting = null
+    this.download = null
 
     this.isPending = false
     this.isClosed = false
     this.isAttached = false
-    this.isIndexer = false
     this.isAdded = false
     this.isRemoved = false
     this.isCoupled = false
 
-    this._onappendBound = this._onappend.bind(this)
-    if (!local) this.core.on('append', this._onappendBound)
+    this._onchangeBound = this._onchange.bind(this)
+
+    if (!local) this.core.on('append', this._onchangeBound)
+    if (!local) this.core.on('download', this._onchangeBound)
 
     this.couple()
   }
 
+  get isIndexer() {
+    return this.weight === 2
+  }
+
   couple() {
     if (this.isCoupled) return
     // if we don't have coupler yet we can't add
@@ -216,7 +225,7 @@ class Writer {
     }
   }
 
-  _onappend() {
+  _onchange() {
     if (this.waiting === null) this.bump()
   }
 
@@ -236,6 +245,8 @@ class Writer {
   detach() {
     if (!this.isAttached) return
     this.isAttached = false
+    this.download.destroy()
+    this.download = null
     this.writers.active.delete(this.id)
     this.removePending()
   }
@@ -258,6 +269,17 @@ class Writer {
     return this.close()
   }
 
+  async views() {
+    const latest = await this.getLatest()
+    if (!latest || !latest.views) return []
+
+    // signedLength for autobase compat
+    return [
+      { ...latest.views.system, signedLength: latest.views.system.length },
+      { ...latest.views.view, signedLength: latest.views.view.length }
+    ]
+  }
+
   async getLatest() {
     if (this.core.length === 0) return null
     const block = await this.core.get(this.core.length - 1)
@@ -273,7 +295,7 @@ class Writer {
     } else {
       this.isAdded = true
       this.isRemoved = info.isRemoved
-      this.isIndexer = info.isIndexer
+      this.weight = info.weight
     }
 
     return info ? info.length : 0
@@ -289,10 +311,21 @@ class Writer {
     let optimistic = false
     let timestamp = 0
 
+    if (this.download) this.download.destroy()
+    this.download = this.core.download({ start: length, end: length + WRITER_PREFETCH })
+
     while (length < this.core.length) {
+      // @todo aim to prefetch when writer added, with read-ahead
+      // download the block if it's not available locally
+      // writer will be added back as pending automatically
+      if (!(await this.core.has(length))) {
+        this.removePending()
+        return null
+      }
+
       const block = await this.core.get(length++)
       const oplog = encoding.decodeOplog(block)
-      const node = createNode(this.core, length, oplog)
+      const node = createNode(this.core, length, this.weight, oplog)
       const b = oplog.batch
 
       // auto correct some batch invariants
@@ -336,6 +369,9 @@ class Writer {
     if (this.core.writable && this.pending !== null) {
       for (let i = 0; i < this.pending.length; i++) {
         const node = this.pending[i]
+
+        node.batch = { start: i, end: this.pending.length - 1 - i }
+
         if (node.length <= length) continue
 
         if (!this.writers.writable && !node.optimistic) {
@@ -409,7 +445,12 @@ class Writer {
       value
     }
 
-    const node = createNode(this.core, this.core.length + this.pending.length + 1, oplog)
+    const node = createNode(
+      this.core,
+      this.core.length + this.pending.length + 1,
+      this.weight,
+      oplog
+    )
 
     this.pending.push(node)
     this.addPending()
@@ -427,7 +468,11 @@ class Writer {
     const s = this.system.bee.head()
     const flushes = this.system.flushes
     this.pending[this.pending.length - 1].views = { system: s, flushes, view }
-    for (const node of this.pending) buffers.push(encoding.encodeOplog(node))
+    for (let i = 0; i < this.pending.length; i++) {
+      const node = this.pending[i]
+      node.batch = { start: i, end: this.pending.length - 1 - i }
+      buffers.push(encoding.encodeOplog(node))
+    }
     this.pending = null
     return this.core.append(buffers)
   }
@@ -437,11 +482,12 @@ exports.ActiveWriters = ActiveWriters
 exports.Writer = Writer
 exports.createNode = createNode
 
-function createNode(core, length, oplog) {
+function createNode(core, length, weight, oplog) {
   return {
     core,
     key: core.key,
     length,
+    weight,
     version: oplog.version,
     timestamp: oplog.timestamp,
     links: oplog.links,

package/package.json

@@ -1,10 +1,14 @@
 {
   "name": "autobee",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
   "main": "index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/holepunchto/autobee.git"
+  },
   "files": [
     "index.js",
     "lib/*.js",
@@ -13,7 +17,7 @@
   "scripts": {
     "format": "prettier . --write",
     "lint": "lunte && prettier . --check",
-    "test": "prettier . --check && lunte && node test/all.js",
+    "test": "node test/all.js",
     "test:bare": "bare test/all.js",
     "test:encrypted": "bare test/all.js --encrypt-all",
     "test:generate": "brittle -r test/all.js test/*.js"