autobee 2.0.0-rc.7 → 2.0.0-rc.9

package/encoding/compat.js

@@ -169,7 +169,7 @@ module.exports = function buildAutobaseSchema(schema, DIR) {
       },
       {
         name: 'value',
-        type: 'buffer',
+        type: 'optionalBuffer',
         required: true
       }
     ]

package/encoding/legacy.js

@@ -153,18 +153,18 @@ const Node = {
   preencode(state, m) {
     Clock.preencode(state, m.heads)
     c.uint.preencode(state, m.batch)
-    c.buffer.preencode(state, m.value)
+    c.optionalBuffer.preencode(state, m.value)
   },
   encode(state, m) {
     Clock.encode(state, m.heads)
     c.uint.encode(state, m.batch)
-    c.buffer.encode(state, m.value)
+    c.optionalBuffer.encode(state, m.value)
   },
   decode(state, m) {
     return {
       heads: Clock.decode(state),
       batch: c.uint.decode(state),
-      value: c.buffer.decode(state)
+      value: c.optionalBuffer.decode(state)
     }
   }
 }

package/encoding/spec/autobee/index.js

@@ -97,8 +97,9 @@ const encoding4 = {
 // @autobase-compat/boot-record
 const encoding5 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
         encoding3.preencode(state, m)
         break
@@ -112,8 +113,9 @@ const encoding5 = {
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
         encoding3.encode(state, m)
         break
@@ -236,17 +238,17 @@ const encoding9 = {
   preencode(state, m) {
     encoding1.preencode(state, m.heads)
     c.uint.preencode(state, m.batch)
-    c.buffer.preencode(state, m.value)
+    c.optionalBuffer.preencode(state, m.value)
   },
   encode(state, m) {
     encoding1.encode(state, m.heads)
     c.uint.encode(state, m.batch)
-    c.buffer.encode(state, m.value)
+    c.optionalBuffer.encode(state, m.value)
   },
   decode(state) {
     const r0 = encoding1.decode(state)
     const r1 = c.uint.decode(state)
-    const r2 = c.buffer.decode(state)
+    const r2 = c.optionalBuffer.decode(state)
 
     return {
       heads: r0,
@@ -496,8 +498,9 @@ const encoding18 = {
 // @autobee/system-info
 const encoding19 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
       case 1:
         encoding15.preencode(state, m)
@@ -513,8 +516,9 @@ const encoding19 = {
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
       case 1:
         encoding15.encode(state, m)
@@ -603,8 +607,9 @@ const encoding20 = {
 // @autobee/system-writer
 const encoding21 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 4
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
       case 1:
       case 2:
@@ -619,8 +624,9 @@ const encoding21 = {
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 4
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
       case 1:
       case 2:
@@ -905,8 +911,9 @@ const encoding29 = {
 // @autobee/oplog
 const encoding30 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.preencode(state, v)
+    switch (v) {
       case 0:
         encoding12.preencode(state, m)
         break
@@ -924,8 +931,9 @@ const encoding30 = {
     }
   },
   encode(state, m) {
-    c.uint.encode(state, m.version)
-    switch (m.version) {
+    const v = m.version ?? 3
+    c.uint.encode(state, v)
+    switch (v) {
       case 0:
         encoding12.encode(state, m)
         break

package/encoding/spec/autobee/schema.json

@@ -194,7 +194,7 @@
         {
           "name": "value",
           "required": true,
-          "type": "buffer",
+          "type": "optionalBuffer",
           "version": 1
         }
       ]
@@ -805,5 +805,15 @@
         }
       ]
     }
+  ],
+  "namespaces": [
+    {
+      "name": "autobee",
+      "external": "../../legacy.js"
+    },
+    {
+      "name": "autobase-compat",
+      "external": "../../legacy.js"
+    }
   ]
 }

package/index.js

@@ -589,58 +589,74 @@ module.exports = class Autobee extends ReadyResource {
     }
   }
 
-  async _getOplog(key, length) {
+  async _getOplog(key, length, opts = null) {
     const core = this.openCore(key)
-    await core.ready()
 
-    const target = length >= 0 ? length : core.length
-    if (target === 0) return null
+    try {
+      await core.ready()
 
-    const buf = await core.get(target - 1)
-    let op = encoding.decodeOplog(buf)
+      const target = length >= 0 ? length : core.length
+      if (target === 0) return null
 
-    if (op.version < 3) {
-      op = await this._inflateLegacyOplog(buf, core, target - 1)
-    }
+      const buf = await core.get(target - 1, opts)
+      if (buf === null) return null
 
-    await core.close()
+      let op = encoding.decodeOplog(buf)
 
-    if (buf === null) return null
+      // legacy nodes always inflate, but only indexers carry views - callers
+      // that need system info must check op.views
+      if (op.version < 3) {
+        op = await this._inflateLegacyOplog(buf, core, target - 1, opts)
+      }
 
-    return {
-      key,
-      length: target,
-      op
+      return {
+        key,
+        length: target,
+        op
+      }
+    } finally {
+      await core.close()
     }
   }
 
-  async _inflateLegacyOplog(buf, core, seq) {
+  async _inflateLegacyOplog(buf, core, seq, opts = null) {
     const m = encoding.decodeRawOplog(buf)
+
+    const op = {
+      version: m.version,
+      timestamp: 0,
+      links: m.node.heads,
+      batch: { start: 0, end: m.node.batch - 1 },
+      views: null,
+      optimistic: !!m.optimistic,
+      value: m.node.value
+    }
+
+    if (m.digest === null || m.checkpoint === null) return op
+
     const fetches = []
 
-    fetches.push(m.digest.pointer ? core.get(seq - m.digest.pointer) : buf)
-    fetches.push(m.checkpoint.pointer ? core.get(seq - m.checkpoint.system.checkpointer) : buf)
+    fetches.push(m.digest.pointer ? core.get(seq - m.digest.pointer, opts) : buf)
+    fetches.push(
+      m.checkpoint.pointer ? core.get(seq - m.checkpoint.system.checkpointer, opts) : buf
+    )
 
     const [digestNode, checkpointNode] = await Promise.all(fetches)
+    // a wait: false get we could not satisfy locally
+    if (digestNode === null || checkpointNode === null) return op
 
     const { digest } = encoding.decodeRawOplog(digestNode)
     const { checkpoint } = encoding.decodeRawOplog(checkpointNode)
 
-    return {
-      version: m.version,
-      timestamp: 0,
-      links: m.node.heads,
-      batch: { start: 0, end: m.node.batch - 1 },
-      views: {
-        system: {
-          key: digest.key,
-          length: checkpoint.system.checkpoint.length
-        },
-        flushes: seq
+    op.views = {
+      system: {
+        key: digest.key,
+        length: checkpoint.system.checkpoint.length
       },
-      optimistic: !!m.optimistic,
-      value: m.node.value
+      flushes: seq
     }
+
+    return op
   }
 
   async _update(changes) {
@@ -1129,14 +1145,16 @@ module.exports = class Autobee extends ReadyResource {
 
   async _rebootFromHead(head, trusted, { force = false, wait = true } = {}) {
     const oplog = await this._getOplog(head.key, head.length)
-    if (!oplog) return false
+    if (!oplog) return null
 
     const verified = trusted ? await this._getOplog(trusted.key, trusted.length) : oplog
+    if (!verified) return null
 
-    if (!verified.op.views) return null
+    // legacy nodes from non-indexers have no system info to reboot from
+    if (!oplog.op.views || !verified.op.views) return null
 
     if (!force && verified.op.views.flushes - this.system.flushes < MIN_FF_GAP) {
-      return false
+      return null
     }
 
     const moved = await this._moveTo(batchToHead(verified.op.views.system), {

package/lib/topo.js

@@ -66,17 +66,66 @@ async function rollback(auto, system, verified) {
   }
 }
 
+// nodes we cannot resolve are returned as null rather than dropped, so callers
+// can tell an incomplete replay from a short one
 async function replay(auto) {
-  const ch = auto.system.bee.createChangesStream()
   const replay = []
 
-  for await (const data of ch) {
-    const { ref } = getOplog(data)
-    replay.push(ref)
+  for await (const entry of oplogIterator(auto.system.bee)) {
+    // the walk was cut short, nothing below this point is reachable
+    if (entry === null) {
+      replay.push(null)
+      break
+    }
+
+    for (const ref of entry.refs) replay.push(ref)
   }
 
-  const inflated = await inflate(auto, replay.reverse())
-  return inflated.flatMap((b) => b.batch)
+  // same reasoning as the iterator - never block on what we do not have
+  const inflated = await inflate(auto, replay.reverse(), { wait: false })
+  return inflated.flatMap((b) => (b === null ? [null] : b.batch))
+}
+
+// tolerates legacy nodes instead of asserting
+async function* oplogIterator(bee) {
+  try {
+    for await (const node of bee.createChangesStream({ wait: false })) {
+      const legacy = isLegacyNode(node)
+      const refs = getOplogs(node)
+      if (refs.length) yield { node, refs, legacy }
+    }
+  } catch {
+    yield null // signal data not available
+  }
+}
+
+// legacy nodes carry no isOplog flag so we cannot tell which writer update is
+// the oplog to fetch - yield every candidate until we have a better signal
+function getOplogs(data) {
+  const undo = data.tail || EMPTY_HEAD
+  const oplogs = []
+  const candidates = []
+
+  for (const { keys } of data.batch) {
+    for (const k of keys) {
+      if (k.key[0] !== 1) continue
+
+      let value = null
+      try {
+        value = encoding.decodeSystemWriter(k.key, k.value)
+      } catch {
+        continue
+      }
+
+      // writer was just added, nothing to fetch
+      if (!value.length) continue
+
+      if (value.isOplog) oplogs.push({ undo, oplog: value })
+      else candidates.push({ undo, oplog: value })
+    }
+  }
+
+  return oplogs.length ? oplogs : candidates
 }
 
 async function sort(auto, batch) {
@@ -193,12 +242,19 @@ function cmp(a, b) {
   return a.length < b.length ? -1 : 1
 }
 
-async function inflate(auto, reorder) {
+async function inflate(auto, reorder, opts = null) {
   const promises = []
   const sessions = new Map()
 
   for (let i = 0; i < reorder.length; i++) {
     const ref = reorder[i]
+
+    // marker for a node we could not resolve, pass it through
+    if (ref === null) {
+      promises.push(null)
+      continue
+    }
+
     const id = b4a.toString(ref.oplog.key, 'hex')
     // only seeds the node - reapplication re-resolves from the witness
     const weight = ref.oplog.weight
@@ -215,7 +271,7 @@ async function inflate(auto, reorder) {
       sessions.set(id, core)
     }
 
-    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i))
+    promises.push(getOplogBatch(ref.undo, core, ref.oplog.length, weight, i, opts))
   }
 
   try {
@@ -225,9 +281,10 @@ async function inflate(auto, reorder) {
   }
 }
 
-async function getOplogBatch(undo, core, length, weight, index) {
+async function getOplogBatch(undo, core, length, weight, index, opts = null) {
   const seq = length - 1
-  const block = await core.get(seq)
+  const block = await core.get(seq, opts)
+  if (block === null) return null
   const oplog = encoding.decodeOplog(block)
   const head = createNode(core, length, weight, oplog)
 
@@ -239,12 +296,18 @@ async function getOplogBatch(undo, core, length, weight, index) {
   const end = seq + head.batch.end // skip head, we have it
 
   for (let i = start; i < end; i++) {
-    remaining.push(core.get(i))
+    remaining.push(core.get(i, opts))
   }
 
   if (remaining.length) {
     const blocks = await Promise.all(remaining)
     for (let i = 0; i < blocks.length; i++) {
+      // keep the hole so the gap in the batch stays visible
+      if (blocks[i] === null) {
+        batch.push(null)
+        continue
+      }
+
       const oplog = encoding.decodeOplog(blocks[i])
       batch.push(createNode(core, i + start + 1, weight, oplog))
     }
@@ -255,8 +318,12 @@ async function getOplogBatch(undo, core, length, weight, index) {
 }
 
 function getLocalBatch(undo, writer, length, weight, index) {
+  if (writer.core.closed) throw new Error('Writer core is closed')
+
   if (length > writer.core.length) {
-    if (!writer.pending) throw new Error('No local nodes')
+    if (!writer.pending) {
+      throw new Error(`No local nodes: looking for ${length}, found ${writer.core.length}`)
+    }
 
     let i = 0
     while (i < writer.pending.length) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autobee",
-  "version": "2.0.0-rc.7",
+  "version": "2.0.0-rc.9",
   "description": "Bee based autobase",
   "license": "Apache-2.0",
   "author": "Holepunch Inc.",
@@ -26,7 +26,7 @@
     "autobee-encryption": "^0.1.2",
     "autobee-wakeup": "^1.1.3",
     "b4a": "^1.8.0",
-    "compact-encoding": "^2.18.0",
+    "compact-encoding": "^3.3.0",
     "hyperbee2": "^2.10.3",
     "hypercore": "^11.33.2",
     "hypercore-id-encoding": "^1.3.0",