hypercore 10.38.2 → 11.0.1

package/lib/core.js

@@ -1,91 +1,154 @@
-const hypercoreCrypto = require('hypercore-crypto')
+const crypto = require('hypercore-crypto')
 const b4a = require('b4a')
 const unslab = require('unslab')
-const Oplog = require('./oplog')
-const BigHeader = require('./big-header')
+const z32 = require('z32')
 const Mutex = require('./mutex')
-const MerkleTree = require('./merkle-tree')
+const { MerkleTree, ReorgBatch } = require('./merkle-tree')
 const BlockStore = require('./block-store')
+const BitInterlude = require('./bit-interlude')
 const Bitfield = require('./bitfield')
 const RemoteBitfield = require('./remote-bitfield')
-const Info = require('./info')
 const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_OPERATION, INVALID_SIGNATURE, INVALID_CHECKSUM } = require('hypercore-errors')
-const m = require('./messages')
 const Verifier = require('./verifier')
 const audit = require('./audit')
+const copyPrologue = require('./copy-prologue')
+const SessionState = require('./session-state')
+const Replicator = require('./replicator')
 
 module.exports = class Core {
-  constructor (header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, sessions, legacy, globalCache, onupdate, onconflict) {
-    this.onupdate = onupdate
-    this.onconflict = onconflict
+  constructor (db, opts = {}) {
+    this.db = db
+    this.storage = null
+    this.replicator = new Replicator(this, opts)
+    this.sessionStates = []
+    this.monitors = []
+    this.activeSessions = 0
+
+    this.id = opts.key ? z32.encode(opts.key) : null
+    this.key = opts.key || null
+    this.discoveryKey = opts.discoveryKey || (opts.key && crypto.discoveryKey(opts.key)) || null
+    this.manifest = null
+    this.opening = null
+    this.closing = null
+    this.exclusive = null
+
     this.preupdate = null
-    this.header = header
-    this.compat = compat
-    this.crypto = crypto
-    this.oplog = oplog
-    this.bigHeader = bigHeader
-    this.tree = tree
-    this.blocks = blocks
-    this.bitfield = bitfield
-    this.verifier = verifier
+    this.header = null
+    this.compat = false
+    this.tree = null
+    this.blocks = null
+    this.bitfield = null
+    this.verifier = null
     this.truncating = 0
     this.updating = false
-    this.closed = false
     this.skipBitfield = null
-    this.active = sessions.length
-    this.sessions = sessions
-    this.globalCache = globalCache
+    this.globalCache = opts.globalCache || null
+    this.autoClose = opts.autoClose !== false
+    this.encryption = null
+    this.onidle = noop
+
+    this.state = null
+    this.opened = false
+    this.destroyed = false
+    this.closed = false
 
-    this._manifestFlushed = !!header.manifest
-    this._maxOplogSize = 65536
-    this._autoFlush = 1
+    this._manifestFlushed = false
+    this._onflush = null
+    this._flushing = null
+    this._activeBatch = null
+    this._bitfield = null
     this._verifies = null
     this._verifiesFlushed = null
-    this._mutex = new Mutex()
-    this._legacy = legacy
+    this._legacy = !!opts.legacy
+
+    this.opening = this._open(opts)
+    this.opening.catch(noop)
+  }
+
+  ready () {
+    return this.opening
+  }
+
+  addMonitor (s) {
+    if (s._monitorIndex >= 0) return
+    s._monitorIndex = this.monitors.push(s) - 1
+  }
+
+  removeMonitor (s) {
+    if (s._monitorIndex < 0) return
+    const head = this.monitors.pop()
+    if (head !== s) this.monitors[(head._monitorIndex = s._monitorIndex)] = head
+    s._monitorIndex = -1
+  }
+
+  emitManifest () {
+    for (let i = this.monitors.length - 1; i >= 0; i--) {
+      this.monitors[i].emit('manifest')
+    }
   }
 
-  static async open (storage, opts = {}) {
-    const oplogFile = storage('oplog')
-    const treeFile = storage('tree')
-    const bitfieldFile = storage('bitfield')
-    const dataFile = storage('data')
-    const headerFile = storage('header')
+  createUserDataStream (opts, session = this.state) {
+    return session.storage.createUserDataStream(opts)
+  }
+
+  allSessions () {
+    const sessions = []
+    for (const state of this.sessionStates) {
+      if (state.sessions.length) sessions.push(...state.sessions)
+    }
+    return sessions
+  }
+
+  hasSession () {
+    return this.activeSessions !== 0
+  }
 
+  checkIfIdle () {
+    if (this.destroyed === true || this.hasSession() === true) return
+    if (this.replicator.idle() === false) return
+    if (this.state === null || this.state.mutex.idle() === false) return
+    this.onidle()
+  }
+
+  async lockExclusive () {
+    if (this.exclusive === null) this.exclusive = new Mutex()
+    await this.exclusive.lock()
+  }
+
+  unlockExclusive () {
+    if (this.exclusive !== null) this.exclusive.unlock()
+  }
+
+  async _open (opts) {
     try {
-      return await this.resume(oplogFile, treeFile, bitfieldFile, dataFile, headerFile, opts)
+      await this._tryOpen(opts)
     } catch (err) {
-      await closeAll(oplogFile, treeFile, bitfieldFile, dataFile, headerFile)
+      this.onidle()
       throw err
     }
+
+    this.opened = true
   }
 
-  static async resume (oplogFile, treeFile, bitfieldFile, dataFile, headerFile, opts) {
+  async _tryOpen (opts) {
+    let storage = await this.db.resume(this.discoveryKey)
+
     let overwrite = opts.overwrite === true
 
     const force = opts.force === true
     const createIfMissing = opts.createIfMissing !== false
-    const crypto = opts.crypto || hypercoreCrypto
     // kill this flag soon
     const legacy = !!opts.legacy
 
-    const oplog = new Oplog(oplogFile, {
-      headerEncoding: m.oplog.header,
-      entryEncoding: m.oplog.entry,
-      readonly: opts.readonly
-    })
-
     // default to true for now if no manifest is provided
     let compat = opts.compat === true || (opts.compat !== false && !opts.manifest)
 
-    let { header, entries } = await oplog.open()
+    let header = storage ? parseHeader(await getCoreInfo(storage)) : null
 
     if (force && opts.key && header && !b4a.equals(header.key, opts.key)) {
       overwrite = true
     }
 
-    const bigHeader = new BigHeader(headerFile)
-
     if (!header || overwrite) {
       if (!createIfMissing) {
         throw STORAGE_EMPTY('No Hypercore is stored here')
@@ -98,11 +161,11 @@ module.exports = class Core {
       }
 
       const keyPair = opts.keyPair || (opts.key ? null : crypto.keyPair())
+
       const defaultManifest = !opts.manifest && (!!opts.compat || !opts.key || !!(keyPair && b4a.equals(opts.key, keyPair.publicKey)))
       const manifest = defaultManifest ? Verifier.defaultSignerManifest(opts.key || keyPair.publicKey) : Verifier.createManifest(opts.manifest)
 
       header = {
-        external: null,
         key: opts.key || (compat ? manifest.signers[0].publicKey : Verifier.manifestHash(manifest)),
         manifest,
         keyPair: keyPair ? { publicKey: keyPair.publicKey, secretKey: keyPair.secretKey || null } : null,
@@ -119,15 +182,30 @@ module.exports = class Core {
         }
       }
 
-      await flushHeader(oplog, bigHeader, header)
-    } else if (header.external) {
-      header = await bigHeader.load(header.external)
+      const discoveryKey = opts.discoveryKey || crypto.discoveryKey(header.key)
+
+      storage = await this.db.create({
+        key: header.key,
+        manifest,
+        keyPair,
+        discoveryKey,
+        userData: opts.userData || [],
+        alias: opts.alias || null
+      })
     }
 
     // unslab the long lived buffers to avoid keeping the slab alive
     header.key = unslab(header.key)
-    header.tree.rootHash = unslab(header.tree.rootHash)
-    header.tree.signature = unslab(header.tree.signature)
+
+    if (header.tree) {
+      header.tree.rootHash = unslab(header.tree.rootHash)
+      header.tree.signature = unslab(header.tree.signature)
+    }
+
+    if (header.keyPair) {
+      header.keyPair.publicKey = unslab(header.keyPair.publicKey)
+      header.keyPair.secretKey = unslab(header.keyPair.secretKey)
+    }
 
     if (header.keyPair) {
       header.keyPair.publicKey = unslab(header.keyPair.publicKey)
@@ -156,551 +234,160 @@ module.exports = class Core {
 
     const prologue = header.manifest ? header.manifest.prologue : null
 
-    const tree = await MerkleTree.open(treeFile, { crypto, prologue, ...header.tree })
-    const bitfield = await Bitfield.open(bitfieldFile)
-    const blocks = new BlockStore(dataFile, tree)
+    const tree = await MerkleTree.open(storage)
+    const bitfield = await Bitfield.open(storage, header.tree.length)
+    const blocks = new BlockStore(storage)
+
+    const treeInfo = {
+      fork: header.tree.fork,
+      signature: header.tree.signature,
+      roots: header.tree.length ? await tree.getRoots(header.tree.length) : [],
+      prologue
+    }
 
     if (overwrite) {
-      await tree.clear()
-      await blocks.clear()
-      await bitfield.clear()
-      entries = []
+      const tx = storage.write()
+      tree.clear(tx)
+      blocks.clear(tx)
+      bitfield.clear(tx)
+      await tx.flush()
     }
 
-    // compat from earlier version that do not store contig length
-    if (header.hints.contiguousLength === 0) {
-      while (bitfield.get(header.hints.contiguousLength)) header.hints.contiguousLength++
+    for await (const { key, value } of storage.createUserDataStream()) {
+      header.userData.push({ key, value: unslab(value) })
     }
 
+    // compat from earlier version that do not store contig length
+    // if (header.hints.contiguousLength === 0) {
+    //   while (bitfield.get(header.hints.contiguousLength)) header.hints.contiguousLength++
+    // }
+
     // to unslab
     if (header.manifest) header.manifest = Verifier.createManifest(header.manifest)
 
     const verifier = header.manifest ? new Verifier(header.key, header.manifest, { crypto, legacy }) : null
 
-    for (const e of entries) {
-      if (e.userData) {
-        updateUserData(header.userData, e.userData.key, e.userData.value)
-      }
-
-      if (e.treeNodes) {
-        for (const node of e.treeNodes) {
-          tree.addNode(node)
-        }
-      }
-
-      if (e.bitfield) {
-        bitfield.setRange(e.bitfield.start, e.bitfield.length, !e.bitfield.drop)
-        updateContig(header, e.bitfield, bitfield)
-      }
-
-      if (e.treeUpgrade) {
-        const batch = await tree.truncate(e.treeUpgrade.length, e.treeUpgrade.fork)
-        batch.ancestors = e.treeUpgrade.ancestors
-        batch.signature = unslab(e.treeUpgrade.signature)
-        addReorgHint(header.hints.reorgs, tree, batch)
-        batch.commit()
-
-        header.tree.length = tree.length
-        header.tree.fork = tree.fork
-        header.tree.rootHash = tree.hash()
-        header.tree.signature = tree.signature
-      }
-    }
+    this.storage = storage
+    this.header = header
+    this.compat = compat
+    this.tree = tree
+    this.blocks = blocks
+    this.bitfield = bitfield
+    this.verifier = verifier
+    this.state = new SessionState(this, null, storage, this.blocks, tree, treeInfo, null)
 
-    for (const entry of header.userData) {
-      entry.value = unslab(entry.value)
-    }
+    if (this.key === null) this.key = this.header.key
+    if (this.discoveryKey === null) this.discoveryKey = crypto.discoveryKey(this.key)
+    if (this.id === null) this.id = z32.encode(this.key)
+    if (this.manifest === null) this.manifest = this.header.manifest
 
-    return new this(header, compat, crypto, oplog, bigHeader, tree, blocks, bitfield, verifier, opts.sessions || [], legacy, opts.globalCache || null, opts.onupdate || noop, opts.onconflict || noop)
+    this._manifestFlushed = !!header.manifest
   }
 
   async audit () {
-    await this._mutex.lock()
+    await this.state.mutex.lock()
 
     try {
-      await this._flushOplog()
-      const corrections = await audit(this)
-      if (corrections.blocks || corrections.tree) await this._flushOplog()
+      const tx = this.state.createWriteBatch()
+
+      // TODO: refactor audit
+      const corrections = await audit(this, tx)
+      if (corrections.blocks || corrections.tree) {
+        await this.state.flushUpdate(tx)
+      }
+
       return corrections
     } finally {
-      await this._mutex.unlock()
+      this.state._unlock()
     }
   }
 
   async setManifest (manifest) {
-    await this._mutex.lock()
+    await this.state.mutex.lock()
 
     try {
       if (manifest && this.header.manifest === null) {
         if (!Verifier.isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match')
-        this._setManifest(Verifier.createManifest(manifest), null)
-        await this._flushOplog()
+
+        const tx = this.state.createWriteBatch()
+        this._setManifest(tx, Verifier.createManifest(manifest), null)
+
+        if (await this.state.flush(tx)) this.replicator.onupgrade()
       }
     } finally {
-      this._mutex.unlock()
+      this.state._unlock()
     }
   }
 
-  _setManifest (manifest, keyPair) {
+  _setManifest (tx, manifest, keyPair) {
     if (!manifest && b4a.equals(keyPair.publicKey, this.header.key)) manifest = Verifier.defaultSignerManifest(this.header.key)
     if (!manifest) return
 
-    const verifier = new Verifier(this.header.key, manifest, { crypto: this.crypto, legacy: this._legacy })
+    const verifier = new Verifier(this.header.key, manifest, { legacy: this._legacy })
+
+    if (verifier.prologue) this.state.prologue = Object.assign({}, verifier.prologue)
 
-    if (verifier.prologue) this.tree.setPrologue(verifier.prologue)
+    this.manifest = this.header.manifest = manifest
+
+    tx.setAuth({
+      key: this.header.key,
+      discoveryKey: this.discoveryKey,
+      manifest,
+      keyPair: this.header.keyPair
+      // TODO: encryptionKey?
+    })
 
-    this.header.manifest = manifest
     this.compat = verifier.compat
     this.verifier = verifier
     this._manifestFlushed = false
 
-    this.onupdate(0b10000, null, null, null)
+    this.replicator.onupgrade()
+    this.emitManifest()
   }
 
-  _shouldFlush () {
-    // TODO: make something more fancy for auto flush mode (like fibonacci etc)
-    if (--this._autoFlush <= 0 || this.oplog.byteLength >= this._maxOplogSize) {
-      this._autoFlush = 4
-      return true
-    }
-
-    if (!this._manifestFlushed && this.header.manifest) {
-      this._manifestFlushed = true
-      return true
-    }
-
-    return false
-  }
-
-  async copyPrologue (src, { additional = [] } = {}) {
-    await this._mutex.lock()
+  async copyPrologue (src) {
+    await this.state.mutex.lock()
 
     try {
-      await src._mutex.lock()
+      await src.mutex.lock()
     } catch (err) {
-      this._mutex.unlock()
+      this.state.mutex.unlock()
       throw err
     }
 
     try {
-      const prologue = this.header.manifest && this.header.manifest.prologue
-      if (!prologue) throw INVALID_OPERATION('No prologue present')
-
-      const srcLength = prologue.length - additional.length
-      const srcBatch = srcLength !== src.tree.length ? await src.tree.truncate(srcLength) : src.tree.batch()
-      const srcRoots = srcBatch.roots.slice(0)
-      const srcByteLength = srcBatch.byteLength
-
-      for (const blk of additional) srcBatch.append(blk)
-
-      if (!b4a.equals(srcBatch.hash(), prologue.hash)) throw INVALID_OPERATION('Source tree is conflicting')
-
-      // all hashes are correct, lets copy
-
-      const entry = {
-        userData: null,
-        treeNodes: srcRoots,
-        treeUpgrade: null,
-        bitfield: null
-      }
-
-      if (additional.length) {
-        await this.blocks.putBatch(srcLength, additional, srcByteLength)
-        entry.treeNodes = entry.treeNodes.concat(srcBatch.nodes)
-        entry.bitfield = {
-          drop: false,
-          start: srcLength,
-          length: additional.length
-        }
-      }
-
-      await this.oplog.append([entry], false)
-      this.tree.addNodes(entry.treeNodes)
-
-      if (this.header.tree.length < srcBatch.length) {
-        this.header.tree.length = srcBatch.length
-        this.header.tree.rootHash = srcBatch.hash()
-
-        this.tree.length = srcBatch.length
-        this.tree.byteLength = srcBatch.byteLength
-        this.tree.roots = srcBatch.roots
-        this.onupdate(0b0001, null, null, null)
-      }
-
-      if (entry.bitfield) {
-        this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)
-        this.onupdate(0, entry.bitfield, null, null)
-      }
-
-      await this._flushOplog()
-
-      // no more additional blocks now and we should be consistant on disk
-      // copy over all existing segments...
-
-      let segmentEnd = 0
-
-      while (segmentEnd < srcLength) {
-        const segmentStart = maximumSegmentStart(segmentEnd, src.bitfield, this.bitfield)
-        if (segmentStart >= srcLength || segmentStart < 0) break
-
-        // max segment is 65536 to avoid running out of memory
-        segmentEnd = Math.min(segmentStart + 65536, srcLength, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
-
-        const treeNodes = await src.tree.getNeededNodes(srcLength, segmentStart, segmentEnd)
-        const bitfield = {
-          drop: false,
-          start: segmentStart,
-          length: segmentEnd - segmentStart
-        }
-
-        const segment = []
-        for (let i = segmentStart; i < segmentEnd; i++) {
-          const blk = await src.blocks.get(i)
-          segment.push(blk)
-        }
-
-        const offset = await src.tree.byteOffset(2 * segmentStart)
-        await this.blocks.putBatch(segmentStart, segment, offset)
-
-        const entry = {
-          userData: null,
-          treeNodes,
-          treeUpgrade: null,
-          bitfield
-        }
-
-        await this.oplog.append([entry], false)
-        this.tree.addNodes(treeNodes)
-        this._setBitfieldRange(bitfield.start, bitfield.length, true)
-        this.onupdate(0, bitfield, null, null)
-        await this._flushOplog()
-      }
-
-      this.header.userData = src.header.userData.slice(0)
-      const contig = Math.min(src.header.hints.contiguousLength, srcBatch.length)
-      if (this.header.hints.contiguousLength < contig) this.header.hints.contiguousLength = contig
-
-      await this._flushOplog()
-    } finally {
-      src._mutex.unlock()
-      this._mutex.unlock()
-    }
-  }
-
-  async flush () {
-    await this._mutex.lock()
-    try {
-      this._manifestFlushed = true
-      this._autoFlush = 4
-      await this._flushOplog()
+      await copyPrologue(src, this)
     } finally {
-      this._mutex.unlock()
+      src.mutex.unlock()
+      this.state.mutex.unlock()
+      this.checkIfIdle()
     }
   }
 
-  async _flushOplog () {
-    // TODO: the apis using this, actually do not need to wait for the bitfields, tree etc to flush
-    // as their mutations are already stored in the oplog. We could potentially just run this in the
-    // background. Might be easier to impl that where it is called instead and keep this one simple.
-    await this.bitfield.flush()
-    await this.tree.flush()
-
-    return flushHeader(this.oplog, this.bigHeader, this.header)
-  }
-
-  _appendBlocks (values) {
-    return this.blocks.putBatch(this.tree.length, values, this.tree.byteLength)
+  get isFlushing () {
+    return !!(this._flushing || this.state._activeBatch)
   }
 
-  async _writeBlock (batch, index, value) {
-    const byteOffset = await batch.byteOffset(index * 2)
-    await this.blocks.put(index, value, byteOffset)
+  flushed () {
+    return this.state.flushed()
   }
 
-  async userData (key, value, flush) {
-    // TODO: each oplog append can set user data, so we should have a way
-    // to just hitch a ride on one of the other ongoing appends?
-    await this._mutex.lock()
-
-    try {
-      let empty = true
-
-      for (const u of this.header.userData) {
-        if (u.key !== key) continue
-        if (value && b4a.equals(u.value, value)) return
-        empty = false
-        break
-      }
-
-      if (empty && !value) return
-
-      const entry = {
-        userData: { key, value },
-        treeNodes: null,
-        treeUpgrade: null,
-        bitfield: null
-      }
-
-      await this.oplog.append([entry], false)
-
-      updateUserData(this.header.userData, key, value)
-
-      if (this._shouldFlush() || flush) await this._flushOplog()
-    } finally {
-      this._mutex.unlock()
+  async _validateCommit (state, treeLength) {
+    if (this.state.length > state.length) {
+      throw new Error('Invalid commit: partial commit') // TODO: partial commit in the future if possible
     }
-  }
-
-  async truncate (length, fork, { signature, keyPair = this.header.keyPair } = {}) {
-    if (this.tree.prologue && length < this.tree.prologue.length) {
-      throw INVALID_OPERATION('Truncation breaks prologue')
-    }
-
-    this.truncating++
-    await this._mutex.lock()
-
-    // upsert compat manifest
-    if (this.verifier === null && keyPair) this._setManifest(null, keyPair)
 
-    try {
-      const batch = await this.tree.truncate(length, fork)
-      if (length > 0) batch.signature = signature || this.verifier.sign(batch, keyPair)
-      await this._truncate(batch, null)
-    } finally {
-      this.truncating--
-      this._mutex.unlock()
-    }
-  }
-
-  async clearBatch () {
-    await this._mutex.lock()
-
-    try {
-      const len = this.bitfield.findFirst(false, this.tree.length)
-      if (len <= this.tree.length) return
-
-      const batch = await this.tree.truncate(this.tree.length, this.tree.fork)
-
-      batch.signature = this.tree.signature // same sig
-
-      const entry = {
-        userData: null,
-        treeNodes: batch.nodes,
-        treeUpgrade: batch,
-        bitfield: {
-          drop: true,
-          start: batch.ancestors,
-          length: len - batch.ancestors
+    if (this.state.length > treeLength) {
+      for (const root of this.state.roots) {
+        const batchRoot = await state.tree.get(root.index)
+        if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
+          throw new Error('Invalid commit: tree conflict')
         }
       }
-
-      await this.oplog.append([entry], false)
-
-      this._setBitfieldRange(batch.ancestors, len - batch.ancestors, false)
-      batch.commit()
-
-      // TODO: (see below todo)
-      await this._flushOplog()
-    } finally {
-      this._mutex.unlock()
     }
-  }
-
-  async clear (start, end, cleared) {
-    await this._mutex.lock()
-
-    try {
-      const entry = {
-        userData: null,
-        treeNodes: null,
-        treeUpgrade: null,
-        bitfield: {
-          start,
-          length: end - start,
-          drop: true
-        }
-      }
-
-      await this.oplog.append([entry], false)
-
-      this._setBitfieldRange(start, end - start, false)
 
-      if (start < this.header.hints.contiguousLength) {
-        this.header.hints.contiguousLength = start
-      }
-
-      start = this.bitfield.lastSet(start) + 1
-      end = this.bitfield.firstSet(end)
-
-      if (end === -1) end = this.tree.length
-      if (start >= end || start >= this.tree.length) return
-
-      const offset = await this.tree.byteOffset(start * 2)
-      const endOffset = await this.tree.byteOffset(end * 2)
-      const length = endOffset - offset
-
-      const before = cleared ? await Info.bytesUsed(this.blocks.storage) : null
-
-      await this.blocks.clear(offset, length)
-
-      const after = cleared ? await Info.bytesUsed(this.blocks.storage) : null
-
-      if (cleared) cleared.blocks = Math.max(before - after, 0)
-
-      this.onupdate(0, entry.bitfield, null, null)
-
-      if (this._shouldFlush()) await this._flushOplog()
-    } finally {
-      this._mutex.unlock()
-    }
-  }
-
-  async purge () {
-    return new Promise((resolve, reject) => {
-      let missing = 4
-      let error = null
-
-      this.oplog.storage.unlink(done)
-      this.tree.storage.unlink(done)
-      this.bitfield.storage.unlink(done)
-      this.blocks.storage.unlink(done)
-
-      function done (err) {
-        if (err) error = err
-        if (--missing) return
-        if (error) reject(error)
-        else resolve()
-      }
-    })
-  }
-
-  async insertBatch (batch, values, { signature, keyPair = this.header.keyPair, pending = false, treeLength = batch.treeLength } = {}) {
-    await this._mutex.lock()
-
-    try {
-      // upsert compat manifest
-      if (this.verifier === null && keyPair) this._setManifest(null, keyPair)
-
-      if (this.tree.fork !== batch.fork) return null
-
-      if (this.tree.length > batch.treeLength) {
-        if (this.tree.length > batch.length) return null // TODO: partial commit in the future if possible
-
-        for (const root of this.tree.roots) {
-          const batchRoot = await batch.get(root.index)
-          if (batchRoot.size !== root.size || !b4a.equals(batchRoot.hash, root.hash)) {
-            return null
-          }
-        }
-      }
-
-      const adding = batch.length - treeLength
-
-      batch.upgraded = !pending && batch.length > this.tree.length
-      batch.treeLength = this.tree.length
-      batch.ancestors = this.tree.length
-      if (batch.upgraded && !pending) batch.signature = signature || this.verifier.sign(batch, keyPair)
-
-      let byteOffset = batch.byteLength
-      for (let i = 0; i < adding; i++) byteOffset -= values[i].byteLength
-
-      if (pending === true) batch.upgraded = false
-
-      const entry = {
-        userData: null,
-        treeNodes: batch.nodes,
-        treeUpgrade: batch.upgraded ? batch : null,
-        bitfield: {
-          drop: false,
-          start: treeLength,
-          length: adding
-        }
-      }
-
-      await this.blocks.putBatch(treeLength, adding < values.length ? values.slice(0, adding) : values, byteOffset)
-      await this.oplog.append([entry], false)
-
-      this._setBitfieldRange(entry.bitfield.start, entry.bitfield.length, true)
-      batch.commit()
-
-      if (batch.upgraded) {
-        this.header.tree.length = batch.length
-        this.header.tree.rootHash = batch.hash()
-        this.header.tree.signature = batch.signature
-      }
-
-      const status = (batch.upgraded ? 0b0001 : 0) | updateContig(this.header, entry.bitfield, this.bitfield)
-      if (!pending) {
-        // we already commit this, and now we signed it, so tell others
-        if (entry.treeUpgrade && treeLength > batch.treeLength) {
-          entry.bitfield.start = batch.treeLength
-          entry.bitfield.length = treeLength - batch.treeLength
-        }
-
-        this.onupdate(status, entry.bitfield, null, null)
-      }
-
-      if (this._shouldFlush()) await this._flushOplog()
-    } finally {
-      this._mutex.unlock()
-    }
-
-    return { length: batch.length, byteLength: batch.byteLength }
-  }
-
-  async append (values, { signature, keyPair = this.header.keyPair, preappend } = {}) {
-    await this._mutex.lock()
-
-    try {
-      // upsert compat manifest
-      if (this.verifier === null && keyPair) this._setManifest(null, keyPair)
-
-      if (preappend) await preappend(values)
-
-      if (!values.length) {
-        return { length: this.tree.length, byteLength: this.tree.byteLength }
-      }
-
-      const batch = this.tree.batch()
-      for (const val of values) batch.append(val)
-
-      // only multisig can have prologue so signature is always present
-      if (this.tree.prologue && batch.length < this.tree.prologue.length) {
-        throw INVALID_OPERATION('Append is not consistent with prologue')
-      }
-
-      batch.signature = signature || this.verifier.sign(batch, keyPair)
-
-      const entry = {
-        userData: null,
-        treeNodes: batch.nodes,
-        treeUpgrade: batch,
-        bitfield: {
-          drop: false,
-          start: batch.ancestors,
-          length: values.length
-        }
-      }
-
-      const byteLength = await this._appendBlocks(values)
-
-      await this.oplog.append([entry], false)
-
-      this._setBitfieldRange(batch.ancestors, batch.length - batch.ancestors, true)
-      batch.commit()
-
-      this.header.tree.length = batch.length
-      this.header.tree.rootHash = batch.hash()
-      this.header.tree.signature = batch.signature
-
-      const status = 0b0001 | updateContig(this.header, entry.bitfield, this.bitfield)
-      this.onupdate(status, entry.bitfield, null, null)
-
-      if (this._shouldFlush()) await this._flushOplog()
-
-      return { length: batch.length, byteLength }
-    } finally {
-      this._mutex.unlock()
+    if (this.verifier === null) {
+      throw INVALID_OPERATION('Cannot commit without manifest') // easier to assert than upsert
     }
   }
 
@@ -715,61 +402,23 @@ module.exports = class Core {
 
     manifest = Verifier.createManifest(manifest) // To unslab
 
-    const verifier = this.verifier || new Verifier(this.header.key, manifest, { crypto: this.crypto, legacy: this._legacy })
+    const verifier = this.verifier || new Verifier(this.header.key, manifest, { legacy: this._legacy })
 
     if (!verifier.verify(batch, batch.signature)) {
       throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
-
-    if (!this.header.manifest) {
-      this.header.manifest = manifest
-      this.compat = verifier.compat
-      this.verifier = verifier
-      this.onupdate(0b10000, null, null, null)
-    }
   }
 
-  async _verifyExclusive ({ batch, bitfield, value, manifest, from }) {
+  async _verifyExclusive ({ batch, bitfield, value, manifest }) {
     this._verifyBatchUpgrade(batch, manifest)
+    if (!batch.commitable()) return false
 
-    await this._mutex.lock()
-
-    try {
-      if (!batch.commitable()) return false
-      this.updating = true
-
-      const entry = {
-        userData: null,
-        treeNodes: batch.nodes,
-        treeUpgrade: batch,
-        bitfield
-      }
-
-      if (this.preupdate !== null) await this.preupdate(batch, this.header.key)
-      if (bitfield) await this._writeBlock(batch, bitfield.start, value)
-
-      await this.oplog.append([entry], false)
-
-      let status = 0b0001
-
-      if (bitfield) {
-        this._setBitfield(bitfield.start, true)
-        status |= updateContig(this.header, bitfield, this.bitfield)
-      }
-
-      batch.commit()
-
-      this.header.tree.fork = batch.fork
-      this.header.tree.length = batch.length
-      this.header.tree.rootHash = batch.hash()
-      this.header.tree.signature = batch.signature
+    if (this.preupdate !== null) await this.preupdate(batch, this.header.key)
 
-      this.onupdate(status, bitfield, value, from)
+    await this.state._verifyBlock(batch, bitfield, value, this.header.manifest ? null : manifest)
 
-      if (this._shouldFlush()) await this._flushOplog()
-    } finally {
-      this.updating = false
-      this._mutex.unlock()
+    if (!batch.upgraded && bitfield) {
+      this.replicator.onhave(bitfield.start, bitfield.length, bitfield.drop)
     }
 
     return true
@@ -778,73 +427,76 @@ module.exports = class Core {
   async _verifyShared () {
     if (!this._verifies.length) return false
 
-    await this._mutex.lock()
+    await this.state.mutex.lock()
+
+    const tx = this.state.createWriteBatch()
 
     const verifies = this._verifies
     this._verifies = null
     this._verified = null
 
     try {
-      const entries = []
-
       for (const { batch, bitfield, value } of verifies) {
         if (!batch.commitable()) continue
 
         if (bitfield) {
-          await this._writeBlock(batch, bitfield.start, value)
+          tx.putBlock(bitfield.start, value)
         }
-
-        entries.push({
-          userData: null,
-          treeNodes: batch.nodes,
-          treeUpgrade: null,
-          bitfield
-        })
       }
 
-      await this.oplog.append(entries, false)
+      const bits = new BitInterlude()
 
       for (let i = 0; i < verifies.length; i++) {
-        const { batch, bitfield, value, manifest, from } = verifies[i]
+        const { batch, bitfield, manifest } = verifies[i]
 
         if (!batch.commitable()) {
           verifies[i] = null // signal that we cannot commit this one
           continue
         }
 
-        let status = 0
-
         if (bitfield) {
-          this._setBitfield(bitfield.start, true)
-          status = updateContig(this.header, bitfield, this.bitfield)
+          bits.setRange(bitfield.start, bitfield.start + 1, true)
         }
 
         // if we got a manifest AND its strictly a non compat one, lets store it
         if (manifest && this.header.manifest === null) {
           if (!Verifier.isValidManifest(this.header.key, manifest)) throw INVALID_CHECKSUM('Manifest hash does not match')
-          this._setManifest(manifest, null)
+          this._setManifest(tx, manifest, null)
         }
 
-        batch.commit()
+        if (batch.commitable()) batch.commit(tx)
+      }
+
+      const ranges = bits.flush(tx, this.bitfield)
+
+      await this.state.flush(tx)
 
-        this.onupdate(status, bitfield, value, from)
+      for (const { start, end, value } of ranges) {
+        this._setBitfieldRanges(start, end, value)
       }
 
-      if (this._shouldFlush()) await this._flushOplog()
+      for (let i = 0; i < verifies.length; i++) {
+        const bitfield = verifies[i] && verifies[i].bitfield
+        if (bitfield) {
+          this.replicator.onhave(bitfield.start, bitfield.length, bitfield.drop)
+          this.updateContiguousLength(bitfield)
+        }
+      }
     } finally {
-      this._mutex.unlock()
+      this.state._clearActiveBatch()
+      this.state.mutex.unlock()
     }
 
     return verifies[0] !== null
   }
 
   async checkConflict (proof, from) {
-    if (this.tree.length < proof.upgrade.length || proof.fork !== this.tree.fork) {
+    if (this.state.length < proof.upgrade.length || proof.fork !== this.state.fork) {
       // out of date this proof - ignore for now
       return false
     }
 
-    const batch = this.tree.verifyFullyRemote(proof)
+    const batch = this.tree.verifyFullyRemote(proof, this.state)
 
     try {
       this._verifyBatchUpgrade(batch, proof.manifest)
@@ -852,20 +504,32 @@ module.exports = class Core {
       return true
     }
 
-    const remoteTreeHash = this.crypto.tree(proof.upgrade.nodes)
-    const localTreeHash = this.crypto.tree(await this.tree.getRoots(proof.upgrade.length))
+    await this.state.mutex.lock()
+
+    try {
+      const tx = this.state.createWriteBatch()
+      if (this.header.manifest === null && proof.manifest) {
+        this._setManifest(tx, proof.manifest, null)
+      }
+
+      await this.state.flush(tx)
+    } finally {
+      this.state.mutex.unlock()
+    }
+
+    const remoteTreeHash = crypto.tree(proof.upgrade.nodes)
+    const localTreeHash = crypto.tree(await this.tree.getRoots(proof.upgrade.length))
 
     if (b4a.equals(localTreeHash, remoteTreeHash)) return false
 
-    await this.onconflict(proof)
+    await this._onconflict(proof)
     return true
   }
 
   async verifyReorg (proof) {
-    const batch = await this.tree.reorg(proof)
-
+    const batch = new ReorgBatch(this.tree, this.state)
+    await this.tree.reorg(proof, batch)
     this._verifyBatchUpgrade(batch, proof.manifest)
-
     return batch
   }
 
@@ -873,9 +537,9 @@ module.exports = class Core {
     // We cannot apply "other forks" atm.
     // We should probably still try and they are likely super similar for non upgrades
     // but this is easy atm (and the above layer will just retry)
-    if (proof.fork !== this.tree.fork) return false
+    if (proof.fork !== this.state.fork) return false
 
-    const batch = await this.tree.verify(proof)
+    const batch = await this.tree.verify(proof, this.state)
     if (!batch.commitable()) return false
 
     const value = (proof.block && proof.block.value) || null
@@ -887,7 +551,9 @@ module.exports = class Core {
       from
     }
 
-    if (batch.upgraded) return this._verifyExclusive(op)
+    if (batch.upgraded) {
+      return this._verifyExclusive(op)
+    }
 
     if (this._verifies !== null) {
       const verifies = this._verifies
@@ -898,102 +564,147 @@ module.exports = class Core {
 
     this._verifies = [op]
     this._verified = this._verifyShared()
+
     return this._verified
   }
 
-  async reorg (batch, from) {
+  async reorg (batch) {
     if (!batch.commitable()) return false
 
     this.truncating++
-    await this._mutex.lock()
 
     try {
-      if (!batch.commitable()) return false
-      await this._truncate(batch, from)
+      await this.state.reorg(batch)
     } finally {
       this.truncating--
-      this._mutex.unlock()
     }
 
     return true
   }
 
-  async _truncate (batch, from) {
-    const entry = {
-      userData: null,
-      treeNodes: batch.nodes,
-      treeUpgrade: batch,
-      bitfield: {
-        drop: true,
-        start: batch.ancestors,
-        length: this.tree.length - batch.ancestors
-      }
+  openSkipBitfield () {
+    if (this.skipBitfield !== null) return this.skipBitfield
+    this.skipBitfield = new RemoteBitfield()
+    const buf = this.bitfield.toBuffer(this.state.length)
+    const bitfield = new Uint32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4)
+    this.skipBitfield.insert(0, bitfield)
+    return this.skipBitfield
+  }
+
+  _setBitfieldRanges (start, end, value) {
+    this.bitfield.setRange(start, end, value)
+    if (this.skipBitfield !== null) this.skipBitfield.setRange(start, end, value)
+  }
+
+  close () {
+    if (!this.closing) this.closing = this._close()
+    return this.closing
+  }
+
+  updateContiguousLength (bitfield) {
+    const contig = updateContigBatch(this.header.hints.contiguousLength, bitfield, this.bitfield)
+
+    if (contig.length !== -1 && contig.length !== this.header.hints.contiguousLength) {
+      this.header.hints.contiguousLength = contig.length
     }
+  }
 
-    await this.oplog.append([entry], false)
+  onappend (tree, bitfield) {
+    this.header.tree = tree
 
-    this._setBitfieldRange(batch.ancestors, this.tree.length - batch.ancestors, false)
-    addReorgHint(this.header.hints.reorgs, this.tree, batch)
-    batch.commit()
+    if (!bitfield) {
+      this.replicator.onupgrade()
+      return
+    }
 
-    const contigStatus = updateContig(this.header, entry.bitfield, this.bitfield)
-    const status = ((batch.length > batch.ancestors) ? 0b0011 : 0b0010) | contigStatus
+    this.replicator.cork()
 
-    this.header.tree.fork = batch.fork
-    this.header.tree.length = batch.length
-    this.header.tree.rootHash = batch.hash()
-    this.header.tree.signature = batch.signature
+    const { start, length, drop } = bitfield
 
-    this.onupdate(status, entry.bitfield, null, from)
+    this._setBitfieldRanges(start, start + length, true)
+    this.updateContiguousLength({ start, length, drop: false })
 
-    // TODO: there is a bug in the merkle tree atm where it cannot handle unflushed
-    // truncates if we append or download anything after the truncation point later on
-    // This is because tree.get checks the truncated flag. We should fix this so we can do
-    // the later flush here as well
-    // if (this._shouldFlush()) await this._flushOplog()
-    await this._flushOplog()
+    this.replicator.onupgrade()
+    this.replicator.onhave(start, length, drop)
+    this.replicator.uncork()
   }
 
-  openSkipBitfield () {
-    if (this.skipBitfield !== null) return this.skipBitfield
-    this.skipBitfield = new RemoteBitfield()
-    const buf = this.bitfield.toBuffer(this.tree.length)
-    const bitfield = new Uint32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4)
-    this.skipBitfield.insert(0, bitfield)
-    return this.skipBitfield
+  ontruncate (tree, { start, length }) {
+    if (tree) this.header.tree = tree
+
+    this.replicator.cork()
+
+    this.replicator.ontruncate(start, length)
+    this.replicator.onhave(start, length, true)
+    this.replicator.onupgrade()
+    this.replicator.uncork()
+
+    for (const sessionState of this.sessionStates) {
+      if (start < sessionState.snapshotCompatLength) sessionState.snapshotCompatLength = start
+    }
+
+    this._setBitfieldRanges(start, start + length, false)
+    this.updateContiguousLength({ start, length, drop: true })
   }
 
-  _setBitfield (index, value) {
-    this.bitfield.set(index, value)
-    if (this.skipBitfield !== null) this.skipBitfield.set(index, value)
+  async _onconflict (proof, from) {
+    await this.replicator.onconflict(from)
+
+    for (let i = this.monitors.length - 1; i >= 0; i--) {
+      const s = this.monitors[i]
+      s.emit('conflict', proof.upgrade.length, proof.fork, proof)
+    }
+
+    const err = new Error('Two conflicting signatures exist for length ' + proof.upgrade.length)
+    await this.closeAllSessions(err)
   }
 
-  _setBitfieldRange (start, length, value) {
-    this.bitfield.setRange(start, length, value)
-    if (this.skipBitfield !== null) this.skipBitfield.setRange(start, length, value)
+  async closeAllSessions (err) {
+    // this.sessions modifies itself when a session closes
+    // This way we ensure we indeed iterate over all sessions
+    const sessions = this.allSessions()
+
+    const all = []
+    for (const s of sessions) all.push(s.close({ error: err, force: false })) // force false or else infinite recursion
+    await Promise.allSettled(all)
+  }
+
+  async destroy () {
+    if (this.destroyed === true) return
+    this.destroyed = true
+
+    if (this.hasSession() === true) throw new Error('Cannot destroy while sessions are open')
+
+    const weakSessions = this.allSessions()
+
+    if (this.replicator) this.replicator.destroy()
+    if (this.state) await this.state.close()
+
+    // close all pending weak sessions...
+    for (const s of weakSessions) s.close().catch(noop)
   }
 
-  async close () {
+  async _close () {
+    if (this.opened === false) await this.opening
+    if (this.hasSession() === true) throw new Error('Cannot close while sessions are open')
+
+    if (this.replicator) await this.replicator.close()
+
+    await this.destroy()
+    if (this.autoClose) await this.storage.store.close()
+
     this.closed = true
-    await this._mutex.destroy()
-    await Promise.allSettled([
-      this.oplog.close(),
-      this.bitfield.close(),
-      this.tree.close(),
-      this.blocks.close(),
-      this.bigHeader.close()
-    ])
   }
 }
 
-function updateContig (header, upd, bitfield) {
+function updateContigBatch (start, upd, bitfield) {
   const end = upd.start + upd.length
 
-  let c = header.hints.contiguousLength
+  let c = start
 
   if (upd.drop) {
     // If we dropped a block in the current contig range, "downgrade" it
-    if (c <= end && c > upd.start) {
+    if (c > upd.start) {
       c = upd.start
     }
   } else {
@@ -1003,104 +714,61 @@ function updateContig (header, upd, bitfield) {
     }
   }
 
-  if (c === header.hints.contiguousLength) {
-    return 0b0000
+  if (c === start) {
+    return {
+      length: -1
+    }
   }
 
-  if (c > header.hints.contiguousLength) {
-    header.hints.contiguousLength = c
-    return 0b0100
+  if (c > start) {
+    return {
+      length: c
+    }
   }
 
-  header.hints.contiguousLength = c
-  return 0b1000
-}
-
-function addReorgHint (list, tree, batch) {
-  if (tree.length === 0 || tree.fork === batch.fork) return
-
-  while (list.length >= 4) list.shift() // 4 here is arbitrary, just want it to be small (hints only)
-  while (list.length > 0) {
-    if (list[list.length - 1].ancestors > batch.ancestors) list.pop()
-    else break
+  return {
+    length: c
   }
-
-  list.push({ from: tree.fork, to: batch.fork, ancestors: batch.ancestors })
 }
 
-function updateUserData (list, key, value) {
-  value = unslab(value)
-
-  for (let i = 0; i < list.length; i++) {
-    if (list[i].key === key) {
-      if (value) list[i].value = value
-      else list.splice(i, 1)
-      return
-    }
+function getDefaultTree () {
+  return {
+    fork: 0,
+    length: 0,
+    rootHash: null,
+    signature: null
   }
-  if (value) list.push({ key, value })
 }
 
-function closeAll (...storages) {
-  let missing = 1
-  let error = null
-
-  return new Promise((resolve, reject) => {
-    for (const s of storages) {
-      missing++
-      s.close(done)
-    }
-
-    done(null)
+function parseHeader (info) {
+  if (!info) return null
 
-    function done (err) {
-      if (err) error = err
-      if (--missing) return
-      if (error) reject(error)
-      else resolve()
+  return {
+    key: info.key,
+    manifest: info.manifest,
+    external: null,
+    keyPair: info.keyPair,
+    userData: [],
+    tree: info.head || getDefaultTree(),
+    hints: {
+      reorgs: [],
+      contiguousLength: 0
     }
-  })
-}
-
-async function flushHeader (oplog, bigHeader, header) {
-  if (header.external) {
-    await bigHeader.flush(header)
-  }
-
-  try {
-    await oplog.flush(header)
-  } catch (err) {
-    if (err.code !== 'OPLOG_HEADER_OVERFLOW') throw err
-    await bigHeader.flush(header)
-    await oplog.flush(header)
   }
 }
 
 function noop () {}
 
-function maximumSegmentStart (start, src, dst) {
-  while (true) {
-    const a = src.firstSet(start)
-    const b = dst.firstUnset(start)
+async function getCoreInfo (storage) {
+  const r = storage.read()
 
-    if (a === -1) return -1
-    if (b === -1) return a
+  const auth = r.getAuth()
+  const head = r.getHead()
 
-    // if dst has the segment, restart
-    if (a < b) {
-      start = b
-      continue
-    }
+  r.tryFlush()
 
-    return a
+  return {
+    ...await auth,
+    head: await head
   }
 }
-
-function minimumSegmentEnd (start, src, dst) {
-  const a = src.firstUnset(start)
-  const b = dst.firstSet(start)
-
-  if (a === -1) return -1
-  if (b === -1) return a
-  return a < b ? a : b
-}