hypercore 10.36.2 → 10.36.4

package/lib/audit.js

@@ -0,0 +1,123 @@
+const hypercoreCrypto = require('hypercore-crypto')
+const flat = require('flat-tree')
+const c = require('compact-encoding')
+const b4a = require('b4a')
+
+// this is optimised for speed over mem atm
+// can be tweaked in the future
+
+module.exports = async function auditCore (core) {
+  const corrections = {
+    tree: 0,
+    blocks: 0
+  }
+
+  const length = core.header.tree.length
+
+  const data = await readFullStorage(core.blocks.storage)
+  const tree = await readFullStorage(core.tree.storage)
+
+  const valid = new Uint8Array(Math.ceil(tree.byteLength / 40))
+  const stack = []
+
+  for (const r of core.tree.roots) {
+    valid[r.index] = 1
+    stack.push(r)
+  }
+
+  while (stack.length > 0) {
+    const node = stack.pop()
+    if ((node.index & 1) === 0) continue
+
+    const [left, right] = flat.children(node.index)
+    const leftNode = getNode(left)
+    const rightNode = getNode(right)
+
+    stack.push(leftNode, rightNode)
+
+    if (valid[node.index]) {
+      const hash = hypercoreCrypto.parent(leftNode, rightNode)
+      if (b4a.equals(hash, node.hash) && node.size === (leftNode.size + rightNode.size)) {
+        valid[leftNode.index] = 1
+        valid[rightNode.index] = 1
+        continue
+      }
+    }
+
+    if (leftNode.size) clearNode(leftNode)
+    if (rightNode.size) clearNode(rightNode)
+  }
+
+  if (corrections.tree) {
+    core.tree.cache.clear()
+  }
+
+  let i = 0
+  let nextOffset = -1
+  while (i < length) {
+    const has = core.bitfield.get(i)
+
+    if (!has) {
+      if (i + 1 === length) break
+      i = core.bitfield.findFirst(true, i + 1)
+      if (i < 0) break
+      nextOffset = -1
+      continue
+    }
+
+    if (nextOffset === -1) {
+      try {
+        nextOffset = await core.tree.byteOffset(i * 2)
+      } catch {
+        core._setBitfield(i, false)
+        corrections.blocks++
+        i++
+        continue
+      }
+    }
+
+    const node = getNode(i * 2)
+    const blk = data.subarray(nextOffset, nextOffset + node.size)
+    const hash = hypercoreCrypto.data(blk)
+
+    nextOffset += blk.byteLength
+
+    if (!b4a.equals(hash, node.hash)) {
+      core._setBitfield(i, false)
+      corrections.blocks++
+    }
+
+    i++
+  }
+
+  return corrections
+
+  function getNode (index) {
+    const state = { start: index * 40, end: index * 40 + 40, buffer: tree }
+    const size = c.uint64.decode(state)
+    const hash = c.fixed32.decode(state)
+    return { index, size, hash }
+  }
+
+  function clearNode (node) {
+    valid[node.index] = 0
+
+    if (node.size) {
+      b4a.fill(tree, 0, node.index * 40, node.index * 40 + 40)
+      core.tree.unflushed.set(node.index, core.tree.blankNode(node.index))
+      corrections.tree++
+    }
+  }
+}
+
+function readFullStorage (storage) {
+  return new Promise((resolve, reject) => {
+    storage.stat((_, st) => {
+      if (!st) return resolve(b4a.alloc(0))
+      storage.read(0, st.size, (err, data) => {
+        if (err) reject(err)
+        else resolve(data)
+      })
+    })
+  })
+}

package/lib/core.js

@@ -11,6 +11,7 @@ const Info = require('./info')
 const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_OPERATION, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
 const m = require('./messages')
 const Verifier = require('./verifier')
+const audit = require('./audit')
 const { createTracer } = require('hypertrace')
 
 module.exports = class Core {
@@ -194,6 +195,19 @@ module.exports = class Core {
     return new this(header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, legacy, opts.onupdate || noop, opts.onconflict || noop)
   }
 
+  async audit () {
+    await this._mutex.lock()
+
+    try {
+      await this._flushOplog()
+      const corrections = await audit(this)
+      if (corrections.blocks || corrections.tree) await this._flushOplog()
+      return corrections
+    } finally {
+      await this._mutex.unlock()
+    }
+  }
+
   async setManifest (manifest) {
     await this._mutex.lock()
 
@@ -601,7 +615,7 @@ module.exports = class Core {
         }
       }
 
-      await this.blocks.putBatch(treeLength, adding > values.length ? values.slice(0, adding) : values, byteOffset)
+      await this.blocks.putBatch(treeLength, adding < values.length ? values.slice(0, adding) : values, byteOffset)
       await this.oplog.append([entry], false)
 
       this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)

package/lib/merkle-tree.js

@@ -4,7 +4,7 @@ const c = require('compact-encoding')
 const Xache = require('xache')
 const b4a = require('b4a')
 const caps = require('./caps')
-const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT } = require('hypercore-errors')
+const { INVALID_PROOF, INVALID_CHECKSUM, INVALID_OPERATION, BAD_ARGUMENT, ASSERTION } = require('hypercore-errors')
 
 const BLANK_HASH = b4a.alloc(32)
 const OLD_TREE = b4a.from([5, 2, 87, 2, 0, 0, 40, 7, 66, 76, 65, 75, 69, 50, 98])
@@ -444,6 +444,10 @@ module.exports = class MerkleTree {
     return true
   }
 
+  blankNode (index) {
+    return blankNode(index)
+  }
+
   get (index, error = true) {
     const c = this.cache.get(index)
     if (c) return c
@@ -763,6 +767,8 @@ async function getByteOffset (tree, index) {
 
     return offset
   }
+
+  throw ASSERTION('Failed to find offset')
 }
 
 // All the methods needed for proof verification

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.36.2",
+  "version": "10.36.4",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {