autobee 1.0.8 → 1.0.10

package/encoding/spec/autobee/index.js

@@ -645,8 +645,33 @@ const encoding21 = {
 // @autobee/link
 const encoding22 = encoding0
 
-// @autobee/batch
+// @autobee/external-batch
 const encoding23 = {
+  preencode(state, m) {
+    c.fixed32.preencode(state, m.key)
+    c.uint.preencode(state, m.start)
+    c.uint.preencode(state, m.length)
+  },
+  encode(state, m) {
+    c.fixed32.encode(state, m.key)
+    c.uint.encode(state, m.start)
+    c.uint.encode(state, m.length)
+  },
+  decode(state) {
+    const r0 = c.fixed32.decode(state)
+    const r1 = c.uint.decode(state)
+    const r2 = c.uint.decode(state)
+
+    return {
+      key: r0,
+      start: r1,
+      length: r2
+    }
+  }
+}
+
+// @autobee/batch
+const encoding24 = {
   preencode(state, m) {
     c.uint.preencode(state, m.start)
     c.uint.preencode(state, m.end)
@@ -667,72 +692,72 @@ const encoding23 = {
 }
 
 // @autobee/views
-const encoding24 = {
+const encoding25 = {
   preencode(state, m) {
-    encoding22.preencode(state, m.system)
     c.uint.preencode(state, m.flushes)
+    encoding23.preencode(state, m.system)
     state.end++ // flags are fixed size
 
-    if (m.view) encoding22.preencode(state, m.view)
+    if (m.view) encoding23.preencode(state, m.view)
   },
   encode(state, m) {
     const flags = m.view ? 1 : 0
 
-    encoding22.encode(state, m.system)
     c.uint.encode(state, m.flushes)
+    encoding23.encode(state, m.system)
     c.uint8.encode(state, flags)
 
-    if (m.view) encoding22.encode(state, m.view)
+    if (m.view) encoding23.encode(state, m.view)
   },
   decode(state) {
-    const r0 = encoding22.decode(state)
-    const r1 = c.uint.decode(state)
+    const r0 = c.uint.decode(state)
+    const r1 = encoding23.decode(state)
     const flags = c.uint8.decode(state)
 
     return {
-      system: r0,
-      flushes: r1,
-      view: (flags & 1) !== 0 ? encoding22.decode(state) : null
+      flushes: r0,
+      system: r1,
+      view: (flags & 1) !== 0 ? encoding23.decode(state) : null
     }
   }
 }
 
 // @autobee/views (inline)
-const encoding24_inline = {
+const encoding25_inline = {
   preencode(state, m) {
-    encoding22.preencode(state, m.system)
     c.uint.preencode(state, m.flushes)
+    encoding23.preencode(state, m.system)
 
-    if (m.view) encoding22.preencode(state, m.view)
+    if (m.view) encoding23.preencode(state, m.view)
   },
   encode(state, m) {
-    encoding22.encode(state, m.system)
     c.uint.encode(state, m.flushes)
+    encoding23.encode(state, m.system)
 
-    if (m.view) encoding22.encode(state, m.view)
+    if (m.view) encoding23.encode(state, m.view)
   },
   decode(state, inlining) {
-    const r0 = encoding22.decode(state)
-    const r1 = c.uint.decode(state)
+    const r0 = c.uint.decode(state)
+    const r1 = encoding23.decode(state)
     const flags = inlining
 
     return {
-      system: r0,
-      flushes: r1,
-      view: (flags & 1) !== 0 ? encoding22.decode(state) : null
+      flushes: r0,
+      system: r1,
+      view: (flags & 1) !== 0 ? encoding23.decode(state) : null
     }
   }
 }
 
 // @autobee/oplog-message-v3
-const encoding25 = {
+const encoding26 = {
   preencode(state, m) {
     c.uint.preencode(state, m.timestamp)
-    encoding25_1.preencode(state, m.links)
+    encoding26_1.preencode(state, m.links)
     state.end++ // max flag is 16 so always one byte
 
-    if (m.batch) encoding23.preencode(state, m.batch)
-    if (m.views) encoding24_inline.preencode(state, m.views)
+    if (m.batch) encoding24.preencode(state, m.batch)
+    if (m.views) encoding25_inline.preencode(state, m.views)
     if (m.value) c.buffer.preencode(state, m.value)
   },
   encode(state, m) {
@@ -742,25 +767,25 @@ const encoding25 = {
     }
 
     c.uint.encode(state, m.timestamp)
-    encoding25_1.encode(state, m.links)
+    encoding26_1.encode(state, m.links)
     c.uint.encode(state, flags)
 
-    if (m.batch) encoding23.encode(state, m.batch)
-    if (m.views) encoding24_inline.encode(state, m.views)
+    if (m.batch) encoding24.encode(state, m.batch)
+    if (m.views) encoding25_inline.encode(state, m.views)
     if (m.value) c.buffer.encode(state, m.value)
   },
   decode(state) {
     const v = c.uint.decode(state)
     const r0 = c.uint.decode(state)
-    const r1 = encoding25_1.decode(state)
+    const r1 = encoding26_1.decode(state)
     const flags = c.uint.decode(state)
 
     return {
       version: v,
       timestamp: r0,
       links: r1,
-      batch: (flags & 1) !== 0 ? encoding23.decode(state) : null,
-      views: (flags & 2) !== 0 ? encoding24_inline.decode(state, flags >>> 2) : null,
+      batch: (flags & 1) !== 0 ? encoding24.decode(state) : null,
+      views: (flags & 2) !== 0 ? encoding25_inline.decode(state, flags >>> 2) : null,
       optimistic: (flags & 8) !== 0,
       value: (flags & 16) !== 0 ? c.buffer.decode(state) : null
     }
@@ -768,7 +793,7 @@ const encoding25 = {
 }
 
 // @autobee/oplog
-const encoding26 = {
+const encoding27 = {
   preencode(state, m) {
     c.uint.preencode(state, m.version)
     switch (m.version) {
@@ -782,7 +807,7 @@ const encoding26 = {
         encoding14.preencode(state, m)
         break
       case 3:
-        encoding25.preencode(state, m)
+        encoding26.preencode(state, m)
         break
       default:
         throw new Error('Unsupported version')
@@ -801,7 +826,7 @@ const encoding26 = {
         encoding14.encode(state, m)
         break
       case 3:
-        encoding25.encode(state, m)
+        encoding26.encode(state, m)
         break
       default:
         throw new Error('Unsupported version')
@@ -825,7 +850,7 @@ const encoding26 = {
         return decoded
       }
       case 3: {
-        const decoded = encoding25.decode(state)
+        const decoded = encoding26.decode(state)
         return decoded
       }
       default:
@@ -835,7 +860,7 @@ const encoding26 = {
 }
 
 // @autobee/manifest-data
-const encoding27 = {
+const encoding28 = {
   preencode(state, m) {
     c.uint.preencode(state, m.version)
     state.end++ // max flag is 2 so always one byte
@@ -869,7 +894,7 @@ const encoding18_3 = c.array(encoding22)
 // @autobee/system-info-v3.indexers, deferred due to recusive use
 const encoding18_4 = encoding18_3
 // @autobee/oplog-message-v3.links, deferred due to recusive use
-const encoding25_1 = encoding18_3
+const encoding26_1 = encoding18_3
 
 function setVersion(v) {
   version = v
@@ -940,16 +965,18 @@ function getEncoding(name) {
       return encoding21
     case '@autobee/link':
       return encoding22
-    case '@autobee/batch':
+    case '@autobee/external-batch':
       return encoding23
-    case '@autobee/views':
+    case '@autobee/batch':
       return encoding24
-    case '@autobee/oplog-message-v3':
+    case '@autobee/views':
       return encoding25
-    case '@autobee/oplog':
+    case '@autobee/oplog-message-v3':
       return encoding26
-    case '@autobee/manifest-data':
+    case '@autobee/oplog':
       return encoding27
+    case '@autobee/manifest-data':
+      return encoding28
     default:
       throw new Error('Encoder not found ' + name)
   }

package/encoding/spec/autobee/schema.json

@@ -526,6 +526,32 @@
         }
       ]
     },
+    {
+      "name": "external-batch",
+      "namespace": "autobee",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "key",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "start",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
     {
       "name": "batch",
       "namespace": "autobee",
@@ -553,21 +579,21 @@
       "flagsPosition": 2,
       "fields": [
         {
-          "name": "system",
+          "name": "flushes",
           "required": true,
-          "type": "@autobee/link",
+          "type": "uint",
           "version": 1
         },
         {
-          "name": "flushes",
+          "name": "system",
           "required": true,
-          "type": "uint",
+          "type": "@autobee/external-batch",
           "version": 1
         },
         {
           "name": "view",
           "required": false,
-          "type": "@autobee/link",
+          "type": "@autobee/external-batch",
           "version": 1
         }
       ]

package/index.js

@@ -79,6 +79,11 @@ module.exports = class Autobee extends ReadyResource {
     this._workingBee = bee
     this._workingView = handlers.open ? handlers.open(this._workingBee, this) : this._workingBee
 
+    this._localSystemStart = 0
+    this._localSystemLength = 0
+    this._localViewStart = 0
+    this._localViewLength = 0
+
     this._appending = []
     this._draining = null
 
@@ -98,6 +103,7 @@ module.exports = class Autobee extends ReadyResource {
     this._interrupting = false
     this._onErrorBound = this._onError.bind(this)
     this._bumpSoonBound = this.bumpSoon.bind(this)
+    this._onGroupUpdateBound = this._onGroupUpdate.bind(this)
 
     this.wakeupCapability = null
     this._wakeup = new AutobeeWakeup(this, handlers)
@@ -137,6 +143,9 @@ module.exports = class Autobee extends ReadyResource {
     await this.bee.ready()
     await this._bootingState
 
+    this._localSystemStart = this.system.bee.context.local.length
+    this._localViewStart = this._workingBee.context.local.length
+
     this.bumpSoon()
   }
 
@@ -269,7 +278,8 @@ module.exports = class Autobee extends ReadyResource {
       }
 
       this._notifyHandler = this.store.notifyGroup(this.wakeupCapability.discoveryKey)
-      this._notifyHandler.on('update', this._bumpSoonBound)
+      this._notifyHandler.on('update', this._onGroupUpdateBound)
+      await this._drainBootHints()
     }
 
     const system = result.system || EMPTY_HEAD
@@ -405,15 +415,37 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _flushWakeup() {
-    const hints = this._wakeup.flush()
+  _onGroupUpdate({ key, length }) {
+    this._wakeup.hint({ key, length })
+    this.bumpSoon()
+  }
 
-    if (this._notifyHandler) {
-      for await (const key of this._notifyHandler.updates({ since: this.previousDrain })) {
-        const hex = b4a.toString(key, 'hex')
-        hints.set(hex, -1)
-      }
+  async _drainBootHints() {
+    if (!this._notifyHandler) return
+
+    const keys = []
+    for await (const key of this._notifyHandler.updates({ since: this.previousDrain })) {
+      keys.push(key)
     }
+    if (!keys.length) return
+
+    // read the lengths straight from storage in one batch instead of opening cores
+    const discoveryKeys = keys.map((key) => crypto.discoveryKey(key))
+    const infos = await this.store.storage.getInfos(discoveryKeys, {
+      auth: false,
+      head: true,
+      hints: false
+    })
+
+    for (let i = 0; i < keys.length; i++) {
+      const info = infos[i]
+      const length = info && info.head ? info.head.length : 0
+      this._wakeup.hint({ key: keys[i], length })
+    }
+  }
+
+  async _flushWakeup() {
+    const hints = this._wakeup.flush()
 
     this.previousDrain = Date.now()
     this.queueWakeupFastForward(hints).catch(noop)
@@ -421,11 +453,7 @@ module.exports = class Autobee extends ReadyResource {
     for (const [hex, length] of hints) {
       const key = b4a.from(hex, 'hex')
       if (this.writers.has(hex)) continue
-      if (length !== -1) {
-        const info = await this.system.get(key)
-        if (info && length <= info.length) continue // stale hint
-      }
-      await this.writers.wakeup(key, length === -1 ? 0 : length)
+      await this.writers.wakeup(key, length)
     }
   }
 
@@ -445,7 +473,8 @@ module.exports = class Autobee extends ReadyResource {
 
     if (buf === null) return null
 
-    return encoding.decodeOplog(buf)
+    const oplog = encoding.decodeOplog(buf)
+    return oplog
   }
 
   _update(changes) {
@@ -492,7 +521,7 @@ module.exports = class Autobee extends ReadyResource {
     const oldLocal = this.local
 
     this.local = newLocal
-    this.writers.rotateLocalWriter(this.local)
+    await this.writers.rotateLocalWriter(this.local)
 
     this._updateLocalCore = null
 
@@ -664,6 +693,8 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _applyBatch(batch, optimistic) {
+    const local = batch[0].core === this.local
+
     const userBatch = []
     for (const node of batch) {
       this.system.addNode(node)
@@ -680,6 +711,11 @@ module.exports = class Autobee extends ReadyResource {
 
     const changed = await this.system.flush(batch, this._workingBee)
 
+    if (local) {
+      this._localSystemLength = this.system.bee.context.local.length - this._localSystemStart
+      this._localViewLength = this._workingBee.context.local.length - this._localViewStart
+    }
+
     await this._storeBoot()
 
     for (const { key, added } of changed) {
@@ -742,8 +778,24 @@ module.exports = class Autobee extends ReadyResource {
   }
 
   async _flushLocal() {
-    // analyze is worth the trade off adding the view here also (technically not needed)
-    await this.writers.flushLocal(this._workingBee.head())
+    await this.writers.flushLocal({
+      flushes: this.system.flushes,
+      system: {
+        key: this.system.bee.context.local.key,
+        start: this._localSystemStart,
+        length: this._localSystemLength
+      },
+      view: {
+        key: this._workingBee.context.local.key,
+        start: this._localViewStart,
+        length: this._localViewLength
+      }
+    })
+
+    this._localSystemStart = this.system.bee.context.local.length
+    this._localSystemLength = 0
+    this._localViewStart = this._workingBee.context.local.length
+    this._localViewLength = 0
   }
 
   moveTo(head, tip) {
@@ -758,9 +810,11 @@ module.exports = class Autobee extends ReadyResource {
     }
 
     const promises = []
+    const heads = []
     for (const [hex, length] of hints) {
       if (length === 0) continue
       const key = b4a.from(hex, 'hex')
+      heads.push({ key, length })
       promises.push(this._getOplog(key, length))
     }
 
@@ -768,24 +822,30 @@ module.exports = class Autobee extends ReadyResource {
     if (this.fastForwarding || this.fastForwardTo) return false
 
     let best = null
+    let bestHead = null
     let bestFlushes = -1
 
-    for (const msg of ops) {
+    for (let i = 0; i < ops.length; i++) {
+      const msg = ops[i]
       if (msg === null) continue
 
       if (msg.views && msg.views.flushes > bestFlushes) {
         bestFlushes = msg.views.flushes
         best = msg.views
+        bestHead = heads[i]
       }
     }
 
     if (best === null || bestFlushes - this.system.flushes < MIN_FF_GAP) return false
 
-    const view = this.bee.checkout(best.view)
+    const view = this.bee.checkout({
+      key: best.view.key,
+      length: best.view.start + best.view.length
+    })
 
     let trusted = null
     try {
-      trusted = await this._handlers.onwakeup(view, this)
+      trusted = await this._handlers.onwakeup(bestHead, view, this)
       if (!trusted || this.fastForwarding || this.fastForwardTo) return false
     } finally {
       view.close()
@@ -793,13 +853,19 @@ module.exports = class Autobee extends ReadyResource {
 
     const oplog = await this._getOplog(trusted.key, trusted.length)
 
-    return this.moveTo(oplog.views.system, {
-      system: best.system,
-      verified: {
-        node: trusted,
-        flushes: oplog.views.flushes
+    return this.moveTo(
+      {
+        key: oplog.views.system.key,
+        length: oplog.views.system.start + oplog.views.system.length
+      },
+      {
+        system: { key: best.system.key, length: best.system.start + best.system.length },
+        verified: {
+          node: trusted,
+          flushes: oplog.views.flushes
+        }
       }
-    })
+    )
   }
 
   async _runFastForward(ff) {

package/lib/writers.js

@@ -67,14 +67,16 @@ class ActiveWriters {
     this.localWriter = new Writer(this, this.auto.system, true, core, b4a.toString(core.key, 'hex'))
     this.localWriter.addPending()
     this.auto.emit('writer', this.localWriter)
+
+    await this.updateLocalState()
   }
 
   clearLocal() {
     return this.localWriter.clear()
   }
 
-  flushLocal(head) {
-    return this.localWriter.flush(head)
+  flushLocal(views) {
+    return this.localWriter.flush(views)
   }
 
   async refresh() {
@@ -88,16 +90,16 @@ class ActiveWriters {
   }
 
   async updateLocalState() {
-    const w = this.localWriter
-    const bootstrapping = w.core.length === 0 && b4a.equals(this.auto.key, w.core.key)
-    const info = await this.auto.system.get(this.auto.local.key)
-    const writable = this.writable
+    await this.localWriter.update()
+  }
 
-    w.weight = info ? info.weight : 0
-    this.writable = info ? !info.isRemoved : bootstrapping
+  _updateLocalState() {
+    const w = this.localWriter
 
+    const writable = w.isAdded && !w.isRemoved
     if (writable === this.writable) return
 
+    this.writable = writable
     this.auto.emit(this.writable ? 'writable' : 'unwritable')
     this.auto.emit('update')
   }
@@ -176,6 +178,8 @@ class Writer {
     this.index = 0
     this.weight = 0
     this.pending = null
+    this.processed = 0
+    this.appendLength = this.core.length + 1
     this.waiting = null
     this.download = null
 
@@ -305,9 +309,12 @@ class Writer {
     if (!latest || !latest.views) return []
 
     // signedLength for autobase compat
+    const system = latest.views.system.start + latest.views.system.length
+    const view = latest.views.view.start + latest.views.view.length
+
     return [
-      { ...latest.views.system, signedLength: latest.views.system.length },
-      { ...latest.views.view, signedLength: latest.views.view.length }
+      { key: latest.views.system.key, length: system, signedLength: system },
+      { key: latest.views.view.key, length: view, signedLength: view }
     ]
   }
 
@@ -322,13 +329,17 @@ class Writer {
     const info = await this.system.get(this.core.key)
 
     if (!info) {
-      this.isAdded = this.system.isGenesis()
+      this.isAdded = this.system.isGenesis() && b4a.equals(this.core.key, this.writers.auto.key)
     } else {
       this.isAdded = true
       this.isRemoved = info.isRemoved
       this.weight = info.weight
     }
 
+    if (this === this.writers.localWriter) {
+      this.writers._updateLocalState()
+    }
+
     return info ? info.length : 0
   }
 
@@ -357,6 +368,9 @@ class Writer {
       const block = await this.core.get(length++)
       const oplog = encoding.decodeOplog(block)
       const node = createNode(this.core, length, this.weight, oplog)
+
+      if (this.writable) node.optimistic = false
+
       const b = oplog.batch
 
       // auto correct some batch invariants
@@ -398,12 +412,18 @@ class Writer {
 
     // handle local writer
     if (this.core.writable && this.pending !== null) {
-      for (let i = 0; i < this.pending.length; i++) {
+      let start = this.processed
+      this.processed = this.pending.length
+
+      for (let i = start; i < this.processed; i++) {
         const node = this.pending[i]
 
-        node.batch = { start: i, end: this.pending.length - 1 - i }
+        if (node.length <= length) {
+          start++
+          continue
+        }
 
-        if (node.length <= length) continue
+        node.batch = { start: i, end: this.pending.length - 1 - i }
 
         if (!this.writers.writable && !node.optimistic) {
           this.writers.clearLocal()
@@ -424,11 +444,14 @@ class Writer {
         }
 
         if (!(await this._isLinked(node))) return null
+      }
 
+      if (this.processed === this.pending.length) {
         this.removePending()
-
-        return this.pending.slice(i)
       }
+
+      if (start === this.processed) return null
+      return this.pending.slice(start, this.processed)
     }
 
     return null
@@ -472,16 +495,11 @@ class Writer {
       links,
       batch,
       views: null,
-      optimistic,
+      optimistic: optimistic && !this.writable,
       value
     }
 
-    const node = createNode(
-      this.core,
-      this.core.length + this.pending.length + 1,
-      this.weight,
-      oplog
-    )
+    const node = createNode(this.core, this.appendLength++, this.weight, oplog)
 
     this.pending.push(node)
     this.addPending()
@@ -490,21 +508,27 @@ class Writer {
   }
 
   clear() {
-    this.pending = null
+    if (!this.pending) return
+
+    if (this.processed === this.pending.length) {
+      this.pending = null
+    } else {
+      this.pending = this.pending.slice(this.processed)
+    }
+
+    this.processed = 0
   }
 
-  flush(view) {
+  flush(views) {
     if (this.pending === null) return Promise.resolve()
     const buffers = []
-    const s = this.system.bee.head()
-    const flushes = this.system.flushes
-    this.pending[this.pending.length - 1].views = { system: s, flushes, view }
-    for (let i = 0; i < this.pending.length; i++) {
+    this.pending[this.pending.length - 1].views = views
+    for (let i = 0; i < this.processed; i++) {
       const node = this.pending[i]
       node.batch = { start: i, end: this.pending.length - 1 - i }
       buffers.push(encoding.encodeOplog(node))
     }
-    this.pending = null
+    this.clear()
     return this.core.append(buffers)
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -24,11 +24,11 @@
   },
   "dependencies": {
     "autobee-encryption": "^0.1.2",
-    "autobee-wakeup": "^1.0.5",
+    "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
     "compact-encoding": "^2.18.0",
-    "hyperbee2": "^2.8.0",
-    "hypercore": "^11.33.0",
+    "hyperbee2": "^2.10.3",
+    "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",
     "hyperschema": "^1.20.1",
     "ready-guard": "^1.0.0",
@@ -38,7 +38,7 @@
   },
   "devDependencies": {
     "brittle": "^3.19.1",
-    "corestore": "^7.10.0",
+    "corestore": "^7.11.0",
     "hypercore-crypto": "^3.6.1",
     "lunte": "^1.6.0",
     "prettier": "^3.8.1",