hypercore 11.16.2 → 11.18.0

package/lib/core.js

@@ -7,7 +7,13 @@ const { MerkleTree, ReorgBatch } = require('./merkle-tree')
 const BitInterlude = require('./bit-interlude')
 const Bitfield = require('./bitfield')
 const RemoteBitfield = require('./remote-bitfield')
-const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
+const {
+  BAD_ARGUMENT,
+  STORAGE_EMPTY,
+  STORAGE_CONFLICT,
+  INVALID_SIGNATURE,
+  INVALID_CHECKSUM
+} = require('hypercore-errors')
 const Verifier = require('./verifier')
 const audit = require('./audit')
 const copyPrologue = require('./copy-prologue')
@@ -15,7 +21,7 @@ const SessionState = require('./session-state')
 const Replicator = require('./replicator')
 
 module.exports = class Core {
-  constructor (db, opts = {}) {
+  constructor(db, opts = {}) {
     this.db = db
     this.storage = null
     this.replicator = new Replicator(this, opts)
@@ -58,33 +64,33 @@ module.exports = class Core {
     this.opening.catch(noop)
   }
 
-  ready () {
+  ready() {
     return this.opening
   }
 
-  addMonitor (s) {
+  addMonitor(s) {
     if (s._monitorIndex >= 0) return
     s._monitorIndex = this.monitors.push(s) - 1
   }
 
-  removeMonitor (s) {
+  removeMonitor(s) {
     if (s._monitorIndex < 0) return
     const head = this.monitors.pop()
     if (head !== s) this.monitors[(head._monitorIndex = s._monitorIndex)] = head
     s._monitorIndex = -1
   }
 
-  emitManifest () {
+  emitManifest() {
     for (let i = this.monitors.length - 1; i >= 0; i--) {
       this.monitors[i].emit('manifest')
     }
   }
 
-  createUserDataStream (opts, session = this.state) {
+  createUserDataStream(opts, session = this.state) {
     return session.storage.createUserDataStream(opts)
   }
 
-  allSessions () {
+  allSessions() {
     const sessions = []
     for (const state of this.sessionStates) {
       if (state.sessions.length) sessions.push(...state.sessions)
@@ -92,27 +98,35 @@ module.exports = class Core {
     return sessions
   }
 
-  hasSession () {
+  hasSession() {
     return this.activeSessions !== 0
   }
 
-  checkIfIdle () {
+  compact() {
+    const compacting = []
+    for (const s of this.sessionStates) {
+      compacting.push(s.storage.compact())
+    }
+    return Promise.all(compacting)
+  }
+
+  checkIfIdle() {
     if (!this.opened || this.destroyed === true || this.hasSession() === true) return
     if (this.replicator.idle() === false) return
     if (this.state === null || this.state.mutex.idle() === false) return
     this.onidle()
   }
 
-  async lockExclusive () {
+  async lockExclusive() {
     if (this.exclusive === null) this.exclusive = new Mutex()
     await this.exclusive.lock()
   }
 
-  unlockExclusive () {
+  unlockExclusive() {
     if (this.exclusive !== null) this.exclusive.unlock()
   }
 
-  async _open (opts) {
+  async _open(opts) {
     try {
       await this._tryOpen(opts)
     } catch (err) {
@@ -123,7 +137,7 @@ module.exports = class Core {
     this.opened = true
   }
 
-  async _tryOpen (opts) {
+  async _tryOpen(opts) {
     if (opts.preopen) await opts.preopen // just a hook to allow exclusive access here...
 
     let storage = await this.db.resume(this.discoveryKey)
@@ -144,7 +158,7 @@ module.exports = class Core {
       overwrite = true
     }
 
-    if (!header && (opts.discoveryKey && !(opts.key || opts.manifest))) {
+    if (!header && opts.discoveryKey && !(opts.key || opts.manifest)) {
       throw STORAGE_EMPTY('No Hypercore is stored here', this.discoveryKey)
     }
 
@@ -161,13 +175,19 @@ module.exports = class Core {
 
       const keyPair = opts.keyPair || (opts.key ? null : crypto.keyPair())
 
-      const defaultManifest = !opts.manifest && (!!opts.compat || !opts.key || !!(keyPair && b4a.equals(opts.key, keyPair.publicKey)))
-      const manifest = defaultManifest ? Verifier.defaultSignerManifest(opts.key || keyPair.publicKey) : Verifier.createManifest(opts.manifest)
+      const defaultManifest =
+        !opts.manifest &&
+        (!!opts.compat || !opts.key || !!(keyPair && b4a.equals(opts.key, keyPair.publicKey)))
+      const manifest = defaultManifest
+        ? Verifier.defaultSignerManifest(opts.key || keyPair.publicKey)
+        : Verifier.createManifest(opts.manifest)
 
       header = {
         key: opts.key || (compat ? manifest.signers[0].publicKey : Verifier.manifestHash(manifest)),
         manifest,
-        keyPair: keyPair ? { publicKey: keyPair.publicKey, secretKey: keyPair.secretKey || null } : null,
+        keyPair: keyPair
+          ? { publicKey: keyPair.publicKey, secretKey: keyPair.secretKey || null }
+          : null,
         frozen: false,
         tree: {
           fork: 0,
@@ -214,7 +234,10 @@ module.exports = class Core {
 
     if (opts.manifest) {
       // if we provide a manifest and no key, verify that the stored key is the same
-      if (!opts.key && !Verifier.isValidManifest(header.key, Verifier.createManifest(opts.manifest))) {
+      if (
+        !opts.key &&
+        !Verifier.isValidManifest(header.key, Verifier.createManifest(opts.manifest))
+      ) {
         throw STORAGE_CONFLICT('Manifest does not hash to provided key', this.discoveryKey)
       }
 
@@ -240,7 +263,9 @@ module.exports = class Core {
       fork: header.tree.fork,
       length: header.tree.length,
       signature: header.tree.signature,
-      roots: header.tree.length ? await MerkleTree.getRootsFromStorage(storage, header.tree.length) : [],
+      roots: header.tree.length
+        ? await MerkleTree.getRootsFromStorage(storage, header.tree.length)
+        : [],
       prologue
     }
 
@@ -273,7 +298,9 @@ module.exports = class Core {
       await tx.flush()
     }
 
-    const verifier = header.manifest ? new Verifier(header.key, header.manifest, { crypto, legacy }) : null
+    const verifier = header.manifest
+      ? new Verifier(header.key, header.manifest, { crypto, legacy })
+      : null
 
     this.storage = storage
     this.header = header
@@ -288,7 +315,7 @@ module.exports = class Core {
     if (this.manifest === null) this.manifest = this.header.manifest
   }
 
-  async audit (opts) {
+  async audit(opts) {
     await this.state.mutex.lock()
 
     try {
@@ -298,12 +325,13 @@ module.exports = class Core {
     }
   }
 
-  async setManifest (manifest) {
+  async setManifest(manifest) {
     await this.state.mutex.lock()
 
     try {
       if (manifest && this.header.manifest === null) {
-        if (!Verifier.isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match', this.discoveryKey)
+        if (!Verifier.isValidManifest(this.header.key, manifest))
+          throw INVALID_CHECKSUM('Manifest hash does not match', this.discoveryKey)
 
         const tx = this.state.createWriteBatch()
         this._setManifest(tx, Verifier.createManifest(manifest), null)
@@ -314,8 +342,9 @@ module.exports = class Core {
     }
   }
 
-  _setManifest (tx, manifest, keyPair) {
-    if (!manifest && b4a.equals(keyPair.publicKey, this.header.key)) manifest = Verifier.defaultSignerManifest(this.header.key)
+  _setManifest(tx, manifest, keyPair) {
+    if (!manifest && b4a.equals(keyPair.publicKey, this.header.key))
+      manifest = Verifier.defaultSignerManifest(this.header.key)
     if (!manifest) return
 
     const verifier = new Verifier(this.header.key, manifest, { legacy: this._legacy })
@@ -339,7 +368,7 @@ module.exports = class Core {
     this.emitManifest()
   }
 
-  async copyPrologue (src) {
+  async copyPrologue(src) {
     await this.state.mutex.lock()
 
     try {
@@ -358,11 +387,11 @@ module.exports = class Core {
     }
   }
 
-  flushed () {
+  flushed() {
     return this.state.flushed()
   }
 
-  async _validateCommit (state, treeLength) {
+  async _validateCommit(state, treeLength) {
     if (this.state.length > state.length) {
       return false // TODO: partial commit and truncation possible in the future
     }
@@ -383,17 +412,25 @@ module.exports = class Core {
     return true
   }
 
-  _verifyBatchUpgrade (batch, manifest) {
+  _verifyBatchUpgrade(batch, manifest) {
     if (!this.header.manifest) {
       // compat, drop at some point
       if (!manifest) manifest = Verifier.defaultSignerManifest(this.header.key)
 
-      if (!manifest || !(Verifier.isValidManifest(this.header.key, manifest) || Verifier.isCompat(this.header.key, manifest))) {
+      if (
+        !manifest ||
+        !(
+          Verifier.isValidManifest(this.header.key, manifest) ||
+          Verifier.isCompat(this.header.key, manifest)
+        )
+      ) {
         throw INVALID_SIGNATURE('Proof contains an invalid manifest', this.discoveryKey) // TODO: proper error type
       }
     }
 
-    const verifier = this.verifier || new Verifier(this.header.key, Verifier.createManifest(manifest), { legacy: this._legacy })
+    const verifier =
+      this.verifier ||
+      new Verifier(this.header.key, Verifier.createManifest(manifest), { legacy: this._legacy })
     if (!verifier.verify(batch, batch.signature)) {
       throw INVALID_SIGNATURE('Proof contains an invalid signature', this.discoveryKey)
     }
@@ -401,14 +438,21 @@ module.exports = class Core {
     return manifest
   }
 
-  async _verifyExclusive ({ batch, bitfield, value, manifest }) {
+  async _verifyExclusive({ batch, bitfield, value, manifest }) {
     manifest = this._verifyBatchUpgrade(batch, manifest)
 
     if (!batch.commitable()) return false
 
     if (this.preupdate !== null) await this.preupdate(batch, this.header.key)
 
-    if (!(await this.state._verifyBlock(batch, bitfield, value, this.header.manifest ? null : manifest))) {
+    if (
+      !(await this.state._verifyBlock(
+        batch,
+        bitfield,
+        value,
+        this.header.manifest ? null : manifest
+      ))
+    ) {
       return false
     }
 
@@ -419,7 +463,7 @@ module.exports = class Core {
     return true
   }
 
-  async _verifyShared () {
+  async _verifyShared() {
     if (!this._verifies.length) return false
 
     await this.state.mutex.lock()
@@ -455,7 +499,8 @@ module.exports = class Core {
 
         // if we got a manifest AND its strictly a non compat one, lets store it
         if (manifest && this.header.manifest === null) {
-          if (!Verifier.isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match', this.discoveryKey)
+          if (!Verifier.isValidManifest(this.header.key, manifest))
+            throw INVALID_CHECKSUM('Manifest hash does not match', this.discoveryKey)
           this._setManifest(tx, manifest, null)
         }
 
@@ -485,7 +530,7 @@ module.exports = class Core {
     return verifies[0] !== null
   }
 
-  async checkConflict (proof, from) {
+  async checkConflict(proof, from) {
     if (this.state.length < proof.upgrade.length || proof.fork !== this.state.fork) {
       // out of date this proof - ignore for now
       return false
@@ -555,12 +600,20 @@ module.exports = class Core {
 
     // tmp log so we can see these
     const id = b4a.toString(this.discoveryKey, 'hex')
-    console.log('[hypercore] conflict detected in ' + id + ' (writable=' + !!this.header.keyPair + ',quorum=' + this.header.manifest.quorum + ')')
+    console.log(
+      '[hypercore] conflict detected in ' +
+        id +
+        ' (writable=' +
+        !!this.header.keyPair +
+        ',quorum=' +
+        this.header.manifest.quorum +
+        ')'
+    )
     await this._onconflict(proof)
     return true
   }
 
-  async verifyReorg (proof) {
+  async verifyReorg(proof) {
     const batch = new ReorgBatch(this.state)
     await MerkleTree.reorg(this.state, proof, batch)
     const manifest = this._verifyBatchUpgrade(batch, proof.manifest)
@@ -581,7 +634,7 @@ module.exports = class Core {
     return batch
   }
 
-  async verify (proof, from) {
+  async verify(proof, from) {
     // We cannot apply "other forks" atm.
     // We should probably still try and they are likely super similar for non upgrades
     // but this is easy atm (and the above layer will just retry)
@@ -616,7 +669,7 @@ module.exports = class Core {
     return this._verified
   }
 
-  async reorg (batch) {
+  async reorg(batch) {
     if (!batch.commitable()) return false
 
     this.truncating++
@@ -630,7 +683,7 @@ module.exports = class Core {
     return true
   }
 
-  openSkipBitfield () {
+  openSkipBitfield() {
     if (this.skipBitfield !== null) return this.skipBitfield
     this.skipBitfield = new RemoteBitfield()
     const buf = this.bitfield.toBuffer(this.state.length)
@@ -639,17 +692,17 @@ module.exports = class Core {
     return this.skipBitfield
   }
 
-  _setBitfieldRanges (start, end, value) {
+  _setBitfieldRanges(start, end, value) {
     this.bitfield.setRange(start, end, value)
     if (this.skipBitfield !== null) this.skipBitfield.setRange(start, end, value)
   }
 
-  close () {
+  close() {
     if (!this.closing) this.closing = this._close()
     return this.closing
   }
 
-  updateContiguousLength (bitfield) {
+  updateContiguousLength(bitfield) {
     const contig = updateContigBatch(this.header.hints.contiguousLength, bitfield, this.bitfield)
 
     if (contig.length !== -1 && contig.length !== this.header.hints.contiguousLength) {
@@ -657,7 +710,7 @@ module.exports = class Core {
     }
   }
 
-  onappend (tree, bitfield) {
+  onappend(tree, bitfield) {
     this.header.tree = tree
 
     if (!bitfield) {
@@ -677,7 +730,7 @@ module.exports = class Core {
     this.replicator.uncork()
   }
 
-  ontruncate (tree, { start, length }) {
+  ontruncate(tree, { start, length }) {
     if (tree) this.header.tree = tree
 
     this.replicator.cork()
@@ -695,7 +748,7 @@ module.exports = class Core {
     this.updateContiguousLength({ start, length, drop: true })
   }
 
-  async _onconflict (proof) {
+  async _onconflict(proof) {
     await this.replicator.onconflict()
 
     for (let i = this.monitors.length - 1; i >= 0; i--) {
@@ -707,7 +760,7 @@ module.exports = class Core {
     await this.closeAllSessions(err)
   }
 
-  async closeAllSessions (err) {
+  async closeAllSessions(err) {
     // this.sessions modifies itself when a session closes
     // This way we ensure we indeed iterate over all sessions
     const sessions = this.allSessions()
@@ -717,7 +770,7 @@ module.exports = class Core {
     await Promise.allSettled(all)
   }
 
-  async destroy () {
+  async destroy() {
     if (this.destroyed === true) return
     this.destroyed = true
 
@@ -732,7 +785,7 @@ module.exports = class Core {
     for (const s of weakSessions) s.close().catch(noop)
   }
 
-  async _close () {
+  async _close() {
     if (this.opened === false) await this.opening
     if (this.hasSession() === true) throw new Error('Cannot close while sessions are open')
 
@@ -745,7 +798,7 @@ module.exports = class Core {
   }
 }
 
-function updateContigBatch (start, upd, bitfield) {
+function updateContigBatch(start, upd, bitfield) {
   const end = upd.start + upd.length
 
   let c = start
@@ -779,7 +832,7 @@ function updateContigBatch (start, upd, bitfield) {
   }
 }
 
-function getDefaultTree () {
+function getDefaultTree() {
   return {
     fork: 0,
     length: 0,
@@ -788,7 +841,7 @@ function getDefaultTree () {
   }
 }
 
-function parseHeader (info) {
+function parseHeader(info) {
   if (!info) return null
 
   return {
@@ -804,9 +857,9 @@ function parseHeader (info) {
   }
 }
 
-function noop () {}
+function noop() {}
 
-async function getCoreInfo (storage) {
+async function getCoreInfo(storage) {
   const r = storage.read()
 
   const auth = r.getAuth()

package/lib/default-encryption.js

@@ -8,7 +8,7 @@ const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
 module.exports = class DefaultEncryption {
   static PADDING = 8
 
-  constructor (encryptionKey, hypercoreKey, opts = {}) {
+  constructor(encryptionKey, hypercoreKey, opts = {}) {
     this.key = encryptionKey
     this.compat = opts.compat === true
 
@@ -18,7 +18,7 @@ module.exports = class DefaultEncryption {
     this.blindingKey = keys.blinding
   }
 
-  static deriveKeys (encryptionKey, hypercoreKey, { block = false, compat = false } = {}) {
+  static deriveKeys(encryptionKey, hypercoreKey, { block = false, compat = false } = {}) {
     const subKeys = b4a.alloc(2 * sodium.crypto_stream_KEYBYTES)
 
     const blockKey = block ? encryptionKey : subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
@@ -26,7 +26,8 @@ module.exports = class DefaultEncryption {
 
     if (!block) {
       if (compat) sodium.crypto_generichash_batch(blockKey, [encryptionKey], hypercoreKey)
-      else sodium.crypto_generichash_batch(blockKey, [DEFAULT_ENCRYPTION, hypercoreKey, encryptionKey])
+      else
+        sodium.crypto_generichash_batch(blockKey, [DEFAULT_ENCRYPTION, hypercoreKey, encryptionKey])
     }
 
     sodium.crypto_generichash(blindingKey, blockKey)
@@ -37,13 +38,13 @@ module.exports = class DefaultEncryption {
     }
   }
 
-  static blockEncryptionKey (hypercoreKey, encryptionKey) {
+  static blockEncryptionKey(hypercoreKey, encryptionKey) {
     const blockKey = b4a.alloc(sodium.crypto_stream_KEYBYTES)
     sodium.crypto_generichash_batch(blockKey, [DEFAULT_ENCRYPTION, hypercoreKey, encryptionKey])
     return blockKey
   }
 
-  static encrypt (index, block, fork, blockKey, blindingKey) {
+  static encrypt(index, block, fork, blockKey, blindingKey) {
     const padding = block.subarray(0, DefaultEncryption.PADDING)
     block = block.subarray(DefaultEncryption.PADDING)
 
@@ -56,26 +57,16 @@ module.exports = class DefaultEncryption {
     // Blind the fork ID, possibly risking reusing the nonce on a reorg of the
     // Hypercore. This is fine as the blinding is best-effort and the latest
     // fork ID shared on replication anyway.
-    sodium.crypto_stream_xor(
-      padding,
-      padding,
-      nonce,
-      blindingKey
-    )
+    sodium.crypto_stream_xor(padding, padding, nonce, blindingKey)
 
     nonce.set(padding, 8)
 
     // The combination of a (blinded) fork ID and a block index is unique for a
     // given Hypercore and is therefore a valid nonce for encrypting the block.
-    sodium.crypto_stream_xor(
-      block,
-      block,
-      nonce,
-      blockKey
-    )
+    sodium.crypto_stream_xor(block, block, nonce, blockKey)
   }
 
-  static decrypt (index, block, blockKey) {
+  static decrypt(index, block, blockKey) {
     const padding = block.subarray(0, DefaultEncryption.PADDING)
     block = block.subarray(DefaultEncryption.PADDING)
 
@@ -84,29 +75,24 @@ module.exports = class DefaultEncryption {
     nonce.set(padding, 8)
 
     // Decrypt the block using the blinded fork ID.
-    sodium.crypto_stream_xor(
-      block,
-      block,
-      nonce,
-      blockKey
-    )
+    sodium.crypto_stream_xor(block, block, nonce, blockKey)
   }
 
-  encrypt (index, block, fork, core) {
+  encrypt(index, block, fork, core) {
     if (core.compat !== this.compat) this._reload(core)
     return DefaultEncryption.encrypt(index, block, fork, this.blockKey, this.blindingKey)
   }
 
-  decrypt (index, block, core) {
+  decrypt(index, block, core) {
     if (core.compat !== this.compat) this._reload(core)
     return DefaultEncryption.decrypt(index, block, this.blockKey)
   }
 
-  padding () {
+  padding() {
     return DefaultEncryption.PADDING
   }
 
-  _reload (core) {
+  _reload(core) {
     const block = b4a.equals(this.key, this.blockKey)
     const keys = DefaultEncryption.deriveKeys(this.key, core.key, { block, compat: core.compat })
 

package/lib/download.js

@@ -1,5 +1,5 @@
 module.exports = class Download {
-  constructor (session, range) {
+  constructor(session, range) {
     this.session = session
     this.range = range
     this.request = null
@@ -8,17 +8,17 @@ module.exports = class Download {
     this.opening.catch(noop)
   }
 
-  ready () {
+  ready() {
     return this.opening
   }
 
-  async _open () {
+  async _open() {
     if (this.session.opened === false) await this.session.opening
     this._download()
     this.opened = true
   }
 
-  async done () {
+  async done() {
     await this.ready()
 
     try {
@@ -29,7 +29,7 @@ module.exports = class Download {
     }
   }
 
-  _download () {
+  _download() {
     const activeRequests = (this.range && this.range.activeRequests) || this.session.activeRequests
     this.request = this.session.core.replicator.addRange(activeRequests, this.range)
     this.request.promise.catch(noop)
@@ -39,22 +39,22 @@ module.exports = class Download {
   /**
    * Deprecated. Use `range.done()`.
    */
-  downloaded () {
+  downloaded() {
     return this.done()
   }
 
-  destroy () {
+  destroy() {
     this._destroyBackground().catch(noop)
   }
 
-  async _destroyBackground () {
+  async _destroyBackground() {
     if (this.opened === false) await this.ready()
     if (this.request.context) this.request.context.detach(this.request)
   }
 }
 
-function noop () {}
+function noop() {}
 
-function isSessionMoved (err) {
+function isSessionMoved(err) {
   return err.code === 'SESSION_MOVED'
 }

package/lib/fully-remote-proof.js

@@ -8,7 +8,7 @@ const crypto = require('hypercore-crypto')
 const flat = require('flat-tree')
 
 class SlimSession {
-  constructor (storage, auth, head, roots) {
+  constructor(storage, auth, head, roots) {
     this.fork = head ? head.fork : 0
     this.roots = roots
     this.length = head ? head.length : 0
@@ -24,7 +24,7 @@ class SlimSession {
 
 module.exports = { verify, proof }
 
-async function verify (storage, buffer, { referrer = null } = {}) {
+async function verify(storage, buffer, { referrer = null } = {}) {
   const state = { buffer, start: 0, end: buffer.byteLength }
 
   const discoveryKey = c.fixed32.decode(state)
@@ -100,7 +100,7 @@ async function verify (storage, buffer, { referrer = null } = {}) {
   return result
 }
 
-async function proof (sender, { index, block = null } = {}) {
+async function proof(sender, { index, block = null } = {}) {
   const treeProof = await sender.proof({
     block: block ? { index, nodes: 0 } : null,
     upgrade: { start: 0, length: sender.length }