hypercore 11.0.9 → 11.0.10

package/index.js

@@ -344,18 +344,20 @@ class Hypercore extends EventEmitter {
     const state = this.state
 
     if (opts.atom) {
-      this.state = await parent.state.createSession(null, checkout, false, opts.atom)
+      this.state = await parent.state.createSession(null, false, opts.atom)
       if (state) state.unref()
     } else if (opts.name) {
       // todo: need to make named sessions safe before ready
       // atm we always copy the state in passCapabilities
-      this.state = await parent.state.createSession(opts.name, checkout, !!opts.overwrite, null)
+      this.state = await parent.state.createSession(opts.name, !!opts.overwrite, null)
       if (state) state.unref() // ref'ed above in setup session
+    }
 
-      if (checkout !== -1 && checkout < this.state.length) {
-        await this.state.truncate(checkout, this.fork)
-      }
-    } else if (this.state === null) {
+    if (this.state && checkout !== -1 && checkout < this.state.length) {
+      await this.state.truncate(checkout, this.fork)
+    }
+
+    if (this.state === null) {
       this.state = this.core.state.ref()
     }
 
@@ -520,9 +522,7 @@ class Hypercore extends EventEmitter {
   get signedLength () {
     if (this.opened === false) return 0
     if (this.state === this.core.state) return this.core.state.length
-    const flushed = this.state.flushedLength()
-
-    return flushed === -1 ? this.state.length : flushed
+    return this.state.flushedLength()
   }
 
   /**

package/lib/core.js

@@ -240,6 +240,7 @@ module.exports = class Core {
 
     const treeInfo = {
       fork: header.tree.fork,
+      length: header.tree.length,
       signature: header.tree.signature,
       roots: header.tree.length ? await tree.getRoots(header.tree.length) : [],
       prologue
@@ -417,7 +418,9 @@ module.exports = class Core {
 
     if (this.preupdate !== null) await this.preupdate(batch, this.header.key)
 
-    await this.state._verifyBlock(batch, bitfield, value, this.header.manifest ? null : manifest)
+    if (!(await this.state._verifyBlock(batch, bitfield, value, this.header.manifest ? null : manifest))) {
+      return false
+    }
 
     if (!batch.upgraded && bitfield) {
       this.replicator.onhave(bitfield.start, bitfield.length, bitfield.drop)

package/lib/merkle-tree.js

@@ -43,6 +43,7 @@ class MerkleTreeBatch {
     this.signature = session.signature
     this.ancestors = session.length
     this.byteLength = session.byteLength
+    this.prologue = session.prologue
     this.hashCached = null
 
     this.committed = false

package/lib/session-state.js

@@ -4,7 +4,7 @@ const assert = require('nanoassert')
 const flat = require('flat-tree')
 const quickbit = require('quickbit-universal')
 
-const { STORAGE_CONFLICT, INVALID_OPERATION, INVALID_SIGNATURE } = require('hypercore-errors')
+const { INVALID_OPERATION, INVALID_SIGNATURE } = require('hypercore-errors')
 
 const Mutex = require('./mutex')
 const Bitfield = require('./bitfield')
@@ -38,10 +38,6 @@ module.exports = class SessionState {
     this.prologue = treeInfo.prologue || null
     this.signature = treeInfo.signature || null
 
-    const deps = this.storage.dependencies
-    this.dependencyLength = deps.length ? deps[deps.length - 1].length : Infinity
-
-    if (treeInfo.length < this.dependencyLength) this.dependencyLength = treeInfo.length
     if (treeInfo.roots.length) this.setRoots(treeInfo.roots)
 
     this.snapshotCompatLength = this.isSnapshot() ? this.length : -1
@@ -61,7 +57,11 @@ module.exports = class SessionState {
   }
 
   isDefault () {
-    return this.core.state === this
+    return this.core.state === this || this.isAtomicDefault()
+  }
+
+  isAtomicDefault () {
+    return !!this.storage.atom && !!this.parent && this.parent.isDefault()
   }
 
   createTreeBatch () {
@@ -85,7 +85,9 @@ module.exports = class SessionState {
 
   flushedLength () {
     if (this.isDefault() || this.isSnapshot()) return this.length
-    return this.dependencyLength
+    const deps = this.storage.dependencies
+    if (deps.length) return deps[deps.length - 1].length
+    return 0
   }
 
   unref () {
@@ -161,11 +163,8 @@ module.exports = class SessionState {
   }
 
   updateDependency (tx, length) {
-    const dependency = updateDependency(this, length)
-    if (dependency) {
-      this.dependencyLength = dependency.length
-      tx.setDependency(dependency)
-    }
+    const dependency = updateDependency(this, length, false)
+    if (dependency) tx.setDependency(dependency)
 
     return dependency
   }
@@ -232,34 +231,19 @@ module.exports = class SessionState {
       signature: this.signature
     }
 
-    if (src.dependencyLength > this.dependencyLength) {
-      this.dependencyLength = src.dependencyLength
-      this.storage.updateDependencyLength(src.dependencyLength)
-    }
-
-    // handle migration
-    if (src.core !== this.core) {
-      if (this.lingers === null) this.lingers = []
-      this.lingers.push(this.storage)
+    const rx = src.storage.read()
+    const dependencyPromise = rx.getDependency()
 
-      this.prologue = src.prologue
-      this.storage = await src.core.state.storage.resumeSession(this.name)
-      this.tree = new MerkleTree(this.storage)
-
-      for (let i = this.core.sessionStates.length - 1; i >= 0; i--) {
-        const state = this.core.sessionStates[i]
-        if (state === this) continue
-        if (state.name === this.name) state._moveToCore(src.core)
-      }
+    rx.tryFlush()
 
-      this._moveToCore(src.core)
+    const dependency = await dependencyPromise
+    if (dependency && dependency.length > this.flushedLength()) {
+      this.storage.updateDependencyLength(dependency.length, false, true)
     }
 
-    const truncated = (bitfield && bitfield.truncated !== -1)
-      ? bitfield.truncated
-      : src.dependencyLength
+    const truncated = bitfield ? bitfield.truncated : -1
 
-    if (truncated < currLength) {
+    if (truncated !== -1 && truncated < currLength) {
       this.ontruncate(tree, truncated, currLength, true)
       if (!bitfield || bitfield.length === 0) return
     }
@@ -296,6 +280,7 @@ module.exports = class SessionState {
     await this.mutex.lock()
 
     try {
+      if (!batch.commitable()) return false
       const tx = this.createWriteBatch()
       this.updating = true
 
@@ -307,9 +292,8 @@ module.exports = class SessionState {
 
       if (manifest) this.core._setManifest(tx, manifest, null)
 
-      if (batch.commitable()) {
-        batch.commit(tx)
-      }
+      assert(batch.commitable(), 'Should still be commitable')
+      batch.commit(tx)
 
       const head = {
         fork: batch.fork,
@@ -335,6 +319,8 @@ module.exports = class SessionState {
       this.updating = false
       this.mutex.unlock()
     }
+
+    return true
   }
 
   async truncate (length, fork, { signature, keyPair } = {}) {
@@ -367,7 +353,7 @@ module.exports = class SessionState {
       this.roots = roots
       this.signature = tree.signature
 
-      if (dependency) this.storage.updateDependencyLength(this.dependencyLength)
+      if (dependency) this.storage.updateDependencyLength(dependency.length, true)
 
       this.ontruncate(tree, tree.length, batch.treeLength, flushed)
     } finally {
@@ -392,7 +378,7 @@ module.exports = class SessionState {
       this.roots = batch.roots
       this.signature = batch.signature
 
-      if (dependency) this.storage.updateDependencyLength(this.dependencyLength)
+      if (dependency) this.storage.updateDependencyLength(dependency.length, true)
 
       this.ontruncate(tree, batch.ancestors, batch.treeLength, flushed)
     } finally {
@@ -415,9 +401,9 @@ module.exports = class SessionState {
     if (tree) storage.setHead(tree)
 
     const truncated = batch.length < this.flushedLength()
-    const dependency = truncated ? updateDependency(this, batch.length) : null
+    const dependency = truncated ? updateDependency(this, batch.length, true) : null
 
-    if (dependency) this.dependencyLength = dependency.length
+    if (dependency) storage.setDependency(dependency)
 
     if (this.isDefault()) {
       await storeBitfieldRange(this.storage, storage, batch.ancestors, batch.treeLength, false)
@@ -436,13 +422,11 @@ module.exports = class SessionState {
 
       this.blocks.clear(tx, start, end)
 
-      const dependency = start < this.flushedLength() ? updateDependency(this, start) : null
-
-      if (dependency) this.dependencyLength = dependency.length
+      const dependency = start < this.flushedLength() ? updateDependency(this, start, true) : null
 
       const flushed = await this.flush()
 
-      if (dependency) this.storage.updateDependencyLength(this.dependencyLength)
+      if (dependency) this.storage.updateDependencyLength(dependency.length, true)
 
       // todo: atomic event handle
       if (this.isDefault() && flushed) {
@@ -578,6 +562,80 @@ module.exports = class SessionState {
     p.length += b.length
   }
 
+  async catchup (length) {
+    assert(!this.isDefault(), 'Cannot catchup signed state') // TODO: make this check better
+
+    await this.mutex.lock()
+
+    try {
+      if (length < this.length) throw INVALID_OPERATION('Can not catchup back in time')
+      if (length === this.length) return
+
+      const origLength = this.length
+
+      let sharedLength = 0
+      for (let i = this.storage.dependencies.length - 1; i >= 0; i--) {
+        const dep = this.storage.dependencies[i]
+        if (dep.dataPointer === this.core.state.storage.core.dataPointer) {
+          sharedLength = dep.length
+          break
+        }
+      }
+
+      const tx = this.createWriteBatch()
+      const rx = this.core.state.storage.read()
+      const rootPromises = []
+
+      for (const root of flat.fullRoots(length * 2)) {
+        rootPromises.push(rx.getTreeNode(root))
+      }
+
+      rx.tryFlush()
+
+      const roots = await Promise.all(rootPromises)
+      const truncating = sharedLength < origLength
+
+      const fork = truncating ? this.fork + 1 : this.fork
+
+      // overwrite it atm, TODO: keep what we can connect to the tree
+      tx.deleteBlockRange(0, -1)
+      tx.deleteTreeNodeRange(0, -1)
+      tx.deleteBitfieldPageRange(0, -1)
+
+      const tree = {
+        fork,
+        length,
+        rootHash: crypto.tree(roots),
+        signature: null
+      }
+
+      tx.setHead(tree)
+
+      // prop a better way to do this
+      const dep = updateDependency(this, sharedLength, true)
+      dep.length = length
+
+      tx.setDependency(dep)
+
+      const flushed = await this.flush()
+
+      // and prop a better way to do this tooo
+      this.storage.updateDependencyLength(sharedLength, true)
+      this.storage.updateDependencyLength(length, false)
+
+      const bitfield = { start: sharedLength, length: tree.length - sharedLength, drop: false }
+
+      this.fork = tree.fork
+      this.roots = roots
+      this.length = tree.length
+
+      if (truncating) this.ontruncate(tree, sharedLength, origLength, flushed)
+      if (sharedLength < length) this.onappend(tree, bitfield, flushed)
+    } finally {
+      this.mutex.unlock()
+    }
+  }
+
   async _overwrite (source, fork, length, treeLength, signature, isDependent, shallow) {
     const blockPromises = []
     const treePromises = []
@@ -675,9 +733,7 @@ module.exports = class SessionState {
 
     if (upgraded) tx.setHead(tree)
 
-    const dependency = isDependent ? updateDependency(this, length) : null
-
-    if (dependency) this.dependencyLength = dependency.length
+    const dependency = isDependent ? updateDependency(this, length, true) : null
 
     const flushed = await this.flush()
 
@@ -686,7 +742,7 @@ module.exports = class SessionState {
     this.length = length
     this.signature = signature
 
-    if (dependency) this.storage.updateDependencyLength(this.dependencyLength)
+    if (dependency) this.storage.updateDependencyLength(dependency.length, true)
 
     return { tree, flushed }
   }
@@ -748,10 +804,7 @@ module.exports = class SessionState {
 
         await state.flush(tx)
 
-        if (dependency) {
-          state.storage.updateDependencyLength(dependency.length)
-          state.dependencyLength = dependency.length
-        }
+        if (dependency) state.storage.updateDependencyLength(dependency.length, false)
       }
 
       const bitfield = { start: treeLength, length: length - treeLength, drop: false }
@@ -799,10 +852,6 @@ module.exports = class SessionState {
     this.core = core
     this.index = this.core.sessionStates.push(this) - 1
 
-    // storage was updated
-    const deps = this.storage.dependencies
-    this.dependencyLength = deps[deps.length - 1].length
-
     for (let i = this.sessions.length - 1; i >= 0; i--) this.sessions[i].transferSession(this.core)
   }
 
@@ -812,9 +861,9 @@ module.exports = class SessionState {
     await this.mutex.lock()
 
     try {
-      if (state.storage && (await state.storage.resumeSession(this.name)) !== null) {
-        throw STORAGE_CONFLICT('Batch has already been created')
-      }
+      // if (state.storage && (await state.storage.resumeSession(this.name)) !== null) {
+      //   throw STORAGE_CONFLICT('Batch has already been created')
+      // }
 
       const treeLength = this.length
 
@@ -822,64 +871,60 @@ module.exports = class SessionState {
         if (this.lingers === null) this.lingers = []
         this.lingers.push(this.storage)
 
+        const resumed = await state.storage.resumeSession(this.name)
+
         const truncation = length < this.length ? await truncateAndFlush(this, length) : null
-        const treeInfo = truncation ? truncation.tree : await state._getTreeHeadAt(this.length)
+        const treeInfo = truncation ? truncation.tree : resumed ? null : await state._getTreeHeadAt(this.length)
 
-        treeInfo.fork = truncation ? this.fork + 1 : this.fork
+        const fork = truncation ? this.fork + 1 : this.fork
 
         // todo: validate treeInfo
 
-        if (!this.storage.atom) {
-          this.storage = await state.storage.createSession(this.name, treeInfo)
+        if (resumed) {
+          this.storage = resumed
         } else {
-          const s = state.storage.atomize(this.storage.atom)
-          this.storage = await s.createSession(this.name, treeInfo)
-          await s.close()
+          treeInfo.fork = fork
+          this.storage = await state.storage.createSession(this.name, treeInfo)
         }
 
         this.tree = new MerkleTree(this.storage)
 
         this.prologue = state.prologue
-        this.fork = treeInfo.fork
+        this.fork = fork
         this.length = length
         this.roots = await this.tree.getRoots(length)
 
         if (truncation) {
-          const { dependency, tree, flushed } = truncation
+          const { dependency, tree } = truncation
 
-          if (dependency) this.storage.updateDependencyLength(this.dependencyLength)
-          this.ontruncate(tree, tree.length, treeLength, flushed)
+          if (dependency) this.storage.updateDependencyLength(dependency.length, true)
+          this.ontruncate(tree, tree.length, treeLength, true)
         }
       }
 
-      if (!this.storage.atom) {
-        for (let i = this.core.sessionStates.length - 1; i >= 0; i--) {
-          const state = this.core.sessionStates[i]
-          if (state === this) continue
-          if (state.name === this.name) state._moveToCore(core)
-        }
+      for (let i = this.core.sessionStates.length - 1; i >= 0; i--) {
+        const state = this.core.sessionStates[i]
+        if (state === this) continue
+        if (state.name === this.name) state._moveToCore(core.core)
       }
 
-      this._moveToCore(core)
+      this._moveToCore(core.core)
     } finally {
       this.mutex.unlock()
     }
   }
 
-  async createSession (name, length, overwrite, atom) {
+  async createSession (name, overwrite, atom) {
     let storage = null
     let treeInfo = null
 
     if (!atom && !overwrite && this.storage) {
       storage = await this.storage.resumeSession(name)
 
-      if (storage !== null) {
-        treeInfo = (await getCoreHead(storage)) || getDefaultTree()
-        if (length !== -1 && treeInfo.length !== length) throw STORAGE_CONFLICT('Different batch stored here')
-      }
+      if (storage !== null) treeInfo = (await getCoreHead(storage)) || getDefaultTree()
     }
 
-    if (length === -1) length = treeInfo ? treeInfo.length : this.length
+    const length = treeInfo ? treeInfo.length : this.length
 
     if (storage === null) {
       treeInfo = await this._getTreeHeadAt(length)
@@ -980,25 +1025,16 @@ async function truncateAndFlush (s, length) {
   }
 }
 
-function updateDependency (state, length) {
-  const dependency = findDependency(state.storage, length)
-  if (dependency === null) return null // skip default state and overlays of default
+function updateDependency (state, length, truncated) {
+  const i = state.storage.findDependencyIndex(length, truncated)
+  if (i === -1) return null // skip default state and overlays of default
 
   return {
-    dataPointer: dependency.dataPointer,
+    dataPointer: state.storage.dependencies[i].dataPointer,
     length
   }
 }
 
-function findDependency (storage, length) {
-  for (let i = storage.dependencies.length - 1; i >= 0; i--) {
-    const dep = storage.dependencies[i]
-    if (dep.length < length) return dep
-  }
-
-  return null
-}
-
 function getDefaultTree () {
   return {
     fork: 0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "11.0.9",
+  "version": "11.0.10",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {