hypercore 11.0.36 → 11.0.38

package/README.md

@@ -40,7 +40,7 @@ Make a new Hypercore instance.
 const core = new Hypercore('./directory') // store data in ./directory
 ```
 
-Alternatively you can pass a [Hypercore Storage](https://github.com/holepunchto/hypercore-storage) or use a [Corestore](https://github.com/holepunchto/corestore) if you want to make many Hypercores efficiently.
+Alternatively you can pass a [Hypercore Storage](https://github.com/holepunchto/hypercore-storage) or use a [Corestore](https://github.com/holepunchto/corestore) if you want to make many Hypercores efficiently. Note that `random-access-storage` is no longer supported.
 
 `key` can be set to a Hypercore key which is a hash of Hypercore's internal auth manifest, describing how to validate the Hypercore. If you do not set this, it will be loaded from storage. If nothing is previously stored, a new auth manifest will be generated giving you local write access to it.
 
@@ -50,15 +50,17 @@ Alternatively you can pass a [Hypercore Storage](https://github.com/holepunchto/
 {
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
-  sparse: true, // enable sparse mode, counting unavailable blocks towards core.length and core.byteLength
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
   encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
-  encryptionKey: k, // optionally pass an encryption key to enable block encryption
+  encryption: { key: buffer }, // the block encryption key
   onwait: () => {}, // hook that is called if gets are waiting for download
   timeout: 0, // wait at max some milliseconds (0 means no timeout)
   writable: true, // disable appends and truncates
-  inflightRange: null // Advanced option. Set to [minInflight, maxInflight] to change the min and max inflight blocks per peer when downloading.
+  inflightRange: null, // Advanced option. Set to [minInflight, maxInflight] to change the min and max inflight blocks per peer when downloading.
+  ongc: (session) => { ... }, // A callback called when the session is garbage collected
+  notDownloadingLinger: 20000, // How many milliseconds to wait after downloading finishes keeping the connection open. Defaults to a random number between 20-40s
+  allowFork: true, // Enables updating core when it forks
 }
 ```
 
@@ -292,7 +294,42 @@ You must close any session you make.
 
 Options are inherited from the parent instance, unless they are re-set.
 
-`options` are the same as in the constructor.
+`options` are the same as in the constructor with the follow additions:
+
+```
+{
+  weak: false // Creates the session as a "weak ref" which closes when all non-weak sessions are closed
+  exclusive: false, // Create a session with exclusive access to the core. Creating an exclusive session on a core with other exclusive sessions, will wait for the session with access to close before the next exclusive session is `ready`
+  checkout: undefined, // A index to checkout the core at. Checkout sessions must be an atom or a named session
+  atom: undefined, // A storage atom for making atomic batch changes across hypercores
+  name: null, // Name the session creating a persisted branch of the core. Still beta so may break in the future
+}
+```
+
+Atoms allow making atomic changes across multiple hypercores. Atoms can be created using a `core`'s `storage` (eg. `const atom = core.state.storage.createAtom()`). Changes made with an atom based session is not persisted until the atom is flushed via `await atom.flush()`, but can be read at any time. When atoms flush, all changes made outside of the atom will be clobbered as the core blocks will now match the atom's blocks. For example:
+
+```js
+const core = new Hypercore('./atom-example')
+await core.ready()
+
+await core.append('block 1')
+
+const atom = core.state.storage.createAtom()
+const atomicSession = core.session({ atom })
+
+await core.append('block 2') // Add blocks not using the atom
+
+await atomicSession.append('atom block 2') // Add different block to atom
+await atom.flush()
+
+console.log((await core.get(core.length - 1)).toString()) // prints 'atom block 2' not 'block 2'
+```
+
+#### `const { byteLength, length } = core.commit(session, opts = {})`
+
+Attempt to apply blocks from the session to the `core`. `core` must be a default core, aka a non-named session.
+
+Returns `null` if committing failed.
 
 #### `const snapshot = core.snapshot([options])`
 
@@ -394,7 +431,13 @@ Buffer containing the optional block encryption key of this core. Will be `null`
 
 #### `core.length`
 
-How many blocks of data are available on this core? If `sparse: false`, this will equal `core.contiguousLength`.
+How many blocks of data are available on this core.
+
+Populated after `ready` has been emitted. Will be `0` before the event.
+
+#### `core.signedLength`
+
+How many blocks of data are available on this core that have been signed by a quorum. This is equal to `core.length` for Hypercores's with a single signer.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 

package/lib/audit.js

@@ -1,121 +1,150 @@
-const hypercoreCrypto = require('hypercore-crypto')
+const crypto = require('hypercore-crypto')
 const flat = require('flat-tree')
 const b4a = require('b4a')
+const { MerkleTree } = require('./merkle-tree')
 
-const BitInterlude = require('./bit-interlude')
+module.exports = async function auditCore (core, { tree = true, blocks = true, bitfield = true, dryRun = false } = {}) {
+  const length = core.state.length
+  const stats = { treeNodes: 0, blocks: 0, bits: 0, droppedTreeNodes: 0, droppedBlocks: 0, droppedBits: 0, corrupt: false }
 
-// this is optimised for speed over mem atm
-// can be tweaked in the future
+  // audit the tree
+  if (tree) {
+    let tx = null
 
-module.exports = async function auditCore (core, storage) {
-  const corrections = {
-    tree: 0,
-    blocks: 0
-  }
+    const roots = await MerkleTree.getRootsFromStorage(core.state.storage, length)
+    const stack = []
 
-  const length = core.header.tree.length
+    for (const r of roots) {
+      if (r === null) {
+        if (!dryRun) {
+          const ptr = core.storage.core
+          await core.storage.store.deleteCore(ptr)
+          return null
+        }
 
-  const bitfield = new BitInterlude()
+        stats.corrupt = true
+      }
 
-  const data = await readAllBlocks(core.storage)
-  const tree = await readAllTreeNodes(core.tree.storage)
+      stack.push(r)
+    }
 
-  const valid = new Uint8Array(Math.ceil(tree.byteLength / 40))
-  const stack = []
+    stats.treeNodes += roots.length
 
-  for (const r of core.tree.roots) {
-    valid[r.index] = 1
-    stack.push(r)
-  }
+    while (stack.length > 0) {
+      const node = stack.pop()
 
-  while (stack.length > 0) {
-    const node = stack.pop()
-    if ((node.index & 1) === 0) continue
+      if ((node.index & 1) === 0) continue
 
-    const [left, right] = flat.children(node.index)
-    const leftNode = tree.get(left)
-    const rightNode = tree.get(right)
+      const [left, right] = flat.children(node.index)
 
-    if (!rightNode && !leftNode) continue
+      const rx = core.state.storage.read()
+      const leftNodePromise = rx.getTreeNode(left)
+      const rightNodePromise = rx.getTreeNode(right)
 
-    stack.push(leftNode, rightNode)
+      rx.tryFlush()
 
-    if (valid[node.index]) {
-      const hash = hypercoreCrypto.parent(leftNode, rightNode)
-      if (b4a.equals(hash, node.hash) && node.size === (leftNode.size + rightNode.size)) {
-        valid[leftNode.index] = 1
-        valid[rightNode.index] = 1
+      const [leftNode, rightNode] = await Promise.all([leftNodePromise, rightNodePromise])
+
+      if (isBadTree(node, leftNode, rightNode)) {
+        if (!tx && !stats.corrupt) tx = core.state.storage.write()
+        const [l, r] = flat.spans(node.index)
+        tx.deleteTreeNodeRange(l, r + 1)
+        stats.droppedTreeNodes++
         continue
       }
+
+      if (!leftNode) continue
+
+      stats.treeNodes += 2
+      stack.push(leftNode, rightNode)
     }
 
-    if (leftNode.size) clearNode(left)
-    if (rightNode.size) clearNode(right)
+    if (tx && !dryRun) await tx.flush()
   }
 
-  let i = 0
-  let nextOffset = -1
-  while (i < length) {
-    const has = core.bitfield.get(i)
-
-    if (!has) {
-      if (i + 1 === length) break
-      i = core.bitfield.findFirst(true, i + 1)
-      if (i < 0) break
-      nextOffset = -1
-      continue
-    }
+  // audit the blocks
+  if (blocks) {
+    let tx = null
 
-    if (nextOffset === -1) {
-      try {
-        nextOffset = await core.tree.byteOffset(i * 2)
-      } catch {
-        storage.deleteBlock(i)
-        bitfield.set(i, false)
-        corrections.blocks++
-        i++
-        continue
+    for await (const block of core.state.storage.createBlockStream()) {
+      if (!core.bitfield.get(block.index)) {
+        if (!tx && !stats.corrupt) tx = core.state.storage.write()
+        tx.deleteBlock(block.index)
+        stats.droppedBlocks++
       }
-    }
 
-    const node = tree.get(i * 2)
-    const blk = data.get(i)
-    const hash = hypercoreCrypto.data(blk)
+      const rx = core.state.storage.read()
+      const treeNodePromise = rx.getTreeNode(2 * block.index)
+
+      rx.tryFlush()
 
-    nextOffset += blk.byteLength
+      const treeNode = await treeNodePromise
+
+      if (isBadBlock(treeNode, block.value)) {
+        if (!tx && !stats.corrupt) tx = core.state.storage.write()
+        tx.deleteBlock(block.index)
+        stats.droppedBlocks++
+        continue
+      }
 
-    if (!b4a.equals(hash, node.hash)) {
-      storage.deleteBlock(i)
-      bitfield.set(i, false)
-      corrections.blocks++
+      stats.blocks++
     }
 
-    i++
+    if (tx && !dryRun) await tx.flush()
   }
 
-  bitfield.flush(storage, core.bitfield)
+  if (bitfield) {
+    let tx = null
 
-  return corrections
+    for (const index of allBits(core.bitfield)) {
+      const rx = core.state.storage.read()
+      const blockPromise = rx.getBlock(index)
 
-  function clearNode (node) {
-    valid[node.index] = 0
-    storage.deleteTreeNode(node.index)
-    corrections.tree++
+      rx.tryFlush()
+
+      const block = await blockPromise
+      if (!block) {
+        stats.droppedBits++
+        if (dryRun) continue
+
+        if (!tx && !stats.corrupt) tx = core.state.storage.write()
+
+        core.bitfield.set(index, false)
+
+        const page = core.bitfield.getBitfield(index)
+        if (page.bitfield) tx.setBitfieldPage(page.index, page.bitfield)
+        else tx.deleteBitfieldPage(page.idnex)
+        continue
+      }
+
+      stats.bits++
+    }
+
+    if (tx && !dryRun) await tx.flush()
   }
+
+  return stats
 }
 
-async function readAllBlocks (storage) {
-  const data = new Map()
-  for await (const block of storage.createBlockStream()) {
-    data.set(block.index, block.value)
-  }
-  return data
+function isBadBlock (node, block) {
+  if (!node) return true
+  const hash = crypto.data(block)
+  return !b4a.equals(hash, node.hash) || node.size !== block.byteLength
 }
 
-async function readAllTreeNodes (storage) {
-  const nodes = new Map()
-  for await (const node of storage.createTreeNodeStream()) {
-    nodes.set(node.index, node)
+function isBadTree (parent, left, right) {
+  if (!left && !right) return false
+  if (!left || !right) return true
+  const hash = crypto.parent(left, right)
+  return !b4a.equals(hash, parent.hash) || parent.size !== (left.size + right.size)
+}
+
+function * allBits (bitfield) {
+  let i = 0
+  if (bitfield.get(0)) yield 0
+  while (true) {
+    i = bitfield.findFirst(true, i + 1)
+    if (i === -1) break
+    yield i
   }
-  return nodes
 }

package/lib/core.js

@@ -286,19 +286,11 @@ module.exports = class Core {
     this._manifestFlushed = !!header.manifest
   }
 
-  async audit () {
+  async audit (opts) {
     await this.state.mutex.lock()
 
     try {
-      const tx = this.state.createWriteBatch()
-
-      // TODO: refactor audit
-      const corrections = await audit(this, tx)
-      if (corrections.blocks || corrections.tree) {
-        await this.state.flushUpdate(tx)
-      }
-
-      return corrections
+      return await audit(this, opts)
     } finally {
       this.state._unlock()
     }

package/lib/replicator.js

@@ -351,11 +351,13 @@ class ProofRequest {
   async fulfill () {
     if (this.proof === null) return null
 
-    const proof = await this.proof.settle()
+    const [proof, block] = await Promise.all([this.proof.settle(), this.block])
 
     if (this.manifest) proof.manifest = this.manifest
-    if (this.block) proof.block.value = await this.block
 
+    if (!block && proof.block) return null
+
+    if (block) proof.block.value = block
     return proof
   }
 }

package/lib/session-state.js

@@ -114,16 +114,19 @@ module.exports = class SessionState {
     return this.treeFork === -1 ? this.core.header.tree.fork : this.treeFork
   }
 
-  async getFlushedTreeInfo (storage) {
+  async updateSnapshotStorage (storage) {
     if (!this.atomized || !this.atomized.flushing) return this.treeInfo()
     await this.atomized.flushed()
 
     let rx = storage.read()
     const headPromise = rx.getHead()
     const authPromise = rx.getAuth()
+    const depPromise = rx.getDependency()
 
     rx.tryFlush()
-    const [head, auth] = await Promise.all([headPromise, authPromise])
+    const [head, auth, dep] = await Promise.all([headPromise, authPromise, depPromise])
+
+    storage.setDependencyHead(dep)
 
     const fork = head ? head.fork : 0
     const length = head ? head.length : 0
@@ -188,7 +191,7 @@ module.exports = class SessionState {
 
   async snapshot () {
     const storage = this.storage.snapshot()
-    const treeInfo = await this.getFlushedTreeInfo(storage)
+    const treeInfo = await this.updateSnapshotStorage(storage)
 
     const s = new SessionState(
       this.core,
@@ -258,6 +261,14 @@ module.exports = class SessionState {
     try {
       const currLength = this.length
 
+      // load dependency into memory
+      const rx = this.storage.read()
+      const dependencyPromise = rx.getDependency()
+
+      rx.tryFlush()
+
+      const dependency = await dependencyPromise
+
       this.fork = src.fork
       this.length = src.length
       this.byteLength = src.byteLength
@@ -271,15 +282,7 @@ module.exports = class SessionState {
         signature: this.signature
       }
 
-      const rx = src.storage.read()
-      const dependencyPromise = rx.getDependency()
-
-      rx.tryFlush()
-
-      const dependency = await dependencyPromise
-      if (dependency && dependency.length > this.flushedLength()) {
-        this.storage.updateDependencyLength(dependency.length, false, true)
-      }
+      if (dependency) this.storage.setDependencyHead(dependency)
 
       const truncated = bitfield ? bitfield.truncated : -1
 
@@ -398,7 +401,7 @@ module.exports = class SessionState {
       this.roots = roots
       this.signature = tree.signature
 
-      if (dependency) this.storage.updateDependencyLength(dependency.length, true)
+      if (dependency) this.storage.setDependencyHead(dependency)
 
       this.ontruncate(tree, tree.length, batch.treeLength, flushed)
     } finally {
@@ -424,7 +427,7 @@ module.exports = class SessionState {
       this.roots = batch.roots
       this.signature = batch.signature
 
-      if (dependency) this.storage.updateDependencyLength(dependency.length, true)
+      if (dependency) this.storage.setDependencyHead(dependency)
 
       this.ontruncate(tree, batch.ancestors, batch.treeLength, flushed)
     } finally {
@@ -481,7 +484,7 @@ module.exports = class SessionState {
 
       const flushed = await this.flush()
 
-      if (dependency) this.storage.updateDependencyLength(dependency.length, true)
+      if (dependency) this.storage.setDependencyHead(dependency)
 
       // todo: atomic event handle
       if (this.isDefault() && flushed) {
@@ -658,7 +661,7 @@ module.exports = class SessionState {
       const truncating = sharedLength < origLength
 
       for (const node of roots) {
-        if (node === null) throw new INVALID_OPERATION('Invalid catchup length, tree nodes not available')
+        if (node === null) throw INVALID_OPERATION('Invalid catchup length, tree nodes not available')
       }
 
       const fork = truncating ? this.fork + 1 : this.fork
@@ -685,9 +688,7 @@ module.exports = class SessionState {
 
       const flushed = await this.flush()
 
-      // and prop a better way to do this tooo
-      this.storage.updateDependencyLength(sharedLength, true)
-      this.storage.updateDependencyLength(length, false)
+      this.storage.setDependencyHead(dep)
 
       this.fork = tree.fork
       this.roots = roots
@@ -846,7 +847,7 @@ module.exports = class SessionState {
 
         await state.flush(tx)
 
-        if (dependency) state.storage.updateDependencyLength(dependency.length, false)
+        if (dependency) state.storage.setDependencyHead(dependency)
       }
 
       const bitfield = { start: treeLength, length: length - treeLength, drop: false }
@@ -953,7 +954,7 @@ module.exports = class SessionState {
         if (truncation) {
           const { dependency } = truncation
 
-          if (dependency) this.storage.updateDependencyLength(dependency.length, true)
+          if (dependency) this.storage.setDependencyHead(dependency)
           truncated = { to: treeLength, fork }
         }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.0.36",
+  "version": "11.0.38",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {