hypercore 10.0.0-alpha.35 → 10.0.0-alpha.38

package/index.js

@@ -15,6 +15,7 @@ const Replicator = require('./lib/replicator')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const { ReadStream, WriteStream } = require('./lib/streams')
+const { BAD_ARGUMENT, SESSION_CLOSED, SESSION_NOT_WRITABLE, SNAPSHOT_NOT_AVAILABLE } = require('./lib/errors')
 
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
@@ -39,7 +40,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (!opts) opts = {}
 
     if (!opts.crypto && key && key.byteLength !== 32) {
-      throw new Error('Hypercore key should be 32 bytes')
+      throw BAD_ARGUMENT('Hypercore key should be 32 bytes')
     }
 
     if (!storage) storage = opts.storage
@@ -64,6 +65,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.writable = false
     this.opened = false
     this.closed = false
+    this.snapshotted = !!opts.snapshot
     this.sessions = opts._sessions || [this]
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
@@ -73,7 +75,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opening.catch(noop)
 
     this._preappend = preappend.bind(this)
-    this._snapshot = opts.snapshot || null
+    this._snapshot = null
     this._findingPeers = 0
   }
 
@@ -83,15 +85,40 @@ module.exports = class Hypercore extends EventEmitter {
       while (indent.length < opts.indentationLvl) indent += ' '
     }
 
+    let peers = ''
+    const min = Math.min(this.peers.length, 5)
+
+    for (let i = 0; i < min; i++) {
+      const peer = this.peers[i]
+
+      peers += indent + '    Peer(\n'
+      peers += indent + '      remotePublicKey: ' + opts.stylize(toHex(peer.remotePublicKey), 'string') + '\n'
+      peers += indent + '      remoteLength: ' + opts.stylize(peer.remoteLength, 'number') + '\n'
+      peers += indent + '      remoteFork: ' + opts.stylize(peer.remoteFork, 'number') + '\n'
+      peers += indent + '      remoteCanUpgrade: ' + opts.stylize(peer.remoteCanUpgrade, 'boolean') + '\n'
+      peers += indent + '    )' + '\n'
+    }
+
+    if (this.peers.length > 5) {
+      peers += indent + '  ... and ' + (this.peers.length - 5) + ' more\n'
+    }
+
+    if (peers) peers = '[\n' + peers + indent + '  ]'
+    else peers = '[ ' + opts.stylize(0, 'number') + ' ]'
+
     return this.constructor.name + '(\n' +
-      indent + '  key: ' + opts.stylize((toHex(this.key)), 'string') + '\n' +
+      indent + '  key: ' + opts.stylize(toHex(this.key), 'string') + '\n' +
       indent + '  discoveryKey: ' + opts.stylize(toHex(this.discoveryKey), 'string') + '\n' +
       indent + '  opened: ' + opts.stylize(this.opened, 'boolean') + '\n' +
+      indent + '  closed: ' + opts.stylize(this.closed, 'boolean') + '\n' +
+      indent + '  snapshotted: ' + opts.stylize(this.snapshotted, 'boolean') + '\n' +
       indent + '  writable: ' + opts.stylize(this.writable, 'boolean') + '\n' +
-      indent + '  sessions: ' + opts.stylize(this.sessions.length, 'number') + '\n' +
-      indent + '  peers: [ ' + opts.stylize(this.peers.length, 'number') + ' ]\n' +
       indent + '  length: ' + opts.stylize(this.length, 'number') + '\n' +
       indent + '  byteLength: ' + opts.stylize(this.byteLength, 'number') + '\n' +
+      indent + '  fork: ' + opts.stylize(this.fork, 'number') + '\n' +
+      indent + '  sessions: [ ' + opts.stylize(this.sessions.length, 'number') + ' ]\n' +
+      indent + '  activeRequests: [ ' + opts.stylize(this.activeRequests.length, 'number') + ' ]\n' +
+      indent + '  peers: ' + peers + '\n' +
       indent + ')'
   }
 
@@ -114,7 +141,7 @@ module.exports = class Hypercore extends EventEmitter {
       noiseStream = new NoiseSecretStream(isInitiator, null, opts)
       outerStream = noiseStream.rawStream
     }
-    if (!noiseStream) throw new Error('Invalid stream')
+    if (!noiseStream) throw BAD_ARGUMENT('Invalid stream')
 
     if (!noiseStream.userData) {
       const protocol = new Protomux(noiseStream)
@@ -144,15 +171,15 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
-  snapshot () {
-    return this.session({ snapshot: { length: this.length, byteLength: this.byteLength, fork: this.fork } })
+  snapshot (opts) {
+    return this.session({ ...opts, snapshot: true })
   }
 
   session (opts = {}) {
     if (this.closing) {
       // This makes the closing logic alot easier. If this turns out to be a problem
       // in practive, open an issue and we'll try to make a solution for it.
-      throw new Error('Cannot make sessions on a closing core')
+      throw SESSION_CLOSED('Cannot make sessions on a closing core')
     }
 
     const Clz = opts.class || Hypercore
@@ -170,6 +197,7 @@ module.exports = class Hypercore extends EventEmitter {
 
   _passCapabilities (o) {
     if (!this.auth) this.auth = o.auth
+
     this.crypto = o.crypto
     this.key = o.key
     this.core = o.core
@@ -177,6 +205,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.encryption = o.encryption
     this.writable = !!(this.auth && this.auth.sign)
     this.autoClose = o.autoClose
+
+    if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
   }
 
   async _openFromExisting (from, opts) {
@@ -280,6 +310,19 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  _updateSnapshot () {
+    const prev = this._snapshot
+    const next = this._snapshot = {
+      length: this.core.tree.length,
+      byteLength: this.core.tree.byteLength,
+      fork: this.core.tree.fork,
+      compatLength: this.core.tree.length
+    }
+
+    if (!prev) return true
+    return prev.length !== next.length || prev.fork !== next.fork
+  }
+
   close () {
     if (this.closing) return this.closing
     this.closing = this._close()
@@ -329,6 +372,13 @@ module.exports = class Hypercore extends EventEmitter {
     const noiseStream = protocolStream.noiseStream
     const protocol = noiseStream.userData
 
+    // If the user wants to, we can make this replication run in a session
+    // that way the core wont close "under them" during replication
+    if (opts.session) {
+      const s = this.session()
+      protocolStream.on('close', () => s.close().catch(noop))
+    }
+
     if (this.opened) {
       this.replicator.attachTo(protocol)
     } else {
@@ -355,9 +405,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get fork () {
-    return this._snapshot
-      ? this._snapshot.fork
-      : (this.core === null ? 0 : this.core.tree.fork)
+    return this.core === null ? 0 : this.core.tree.fork
   }
 
   get peers () {
@@ -386,21 +434,33 @@ module.exports = class Hypercore extends EventEmitter {
 
   _oncoreupdate (status, bitfield, value, from) {
     if (status !== 0) {
+      const truncated = (status & 0b10) !== 0
+      const appended = (status & 0b01) !== 0
+
+      if (truncated) {
+        this.replicator.ontruncate(bitfield.start)
+      }
+
       for (let i = 0; i < this.sessions.length; i++) {
-        if ((status & 0b10) !== 0) {
-          if (this.cache) this.cache.clear()
-          this.sessions[i].emit('truncate', bitfield.start, this.core.tree.fork)
+        const s = this.sessions[i]
+
+        if (truncated) {
+          if (s.cache) s.cache.clear()
+          s.emit('truncate', bitfield.start, this.core.tree.fork)
+          // If snapshotted, make sure to update our compat so we can fail gets
+          if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
         }
-        if ((status & 0b01) !== 0) {
-          this.sessions[i].emit('append')
+
+        if (appended) {
+          s.emit('append')
         }
       }
 
-      this.replicator.localUpgrade()
+      this.replicator.onupgrade()
     }
 
     if (bitfield) {
-      this.replicator.broadcastRange(bitfield.start, bitfield.length, bitfield.drop)
+      this.replicator.onhave(bitfield.start, bitfield.length, bitfield.drop)
     }
 
     if (value) {
@@ -457,15 +517,21 @@ module.exports = class Hypercore extends EventEmitter {
 
   async update (opts) {
     if (this.opened === false) await this.opening
+    if (this.closing !== null) return false
 
     // TODO: add an option where a writer can bootstrap it's state from the network also
-    if (this.writable || this.closing !== null) return false
+    if (this.writable) {
+      if (!this.snapshotted) return false
+      return this._updateSnapshot()
+    }
 
     const activeRequests = (opts && opts.activeRequests) || this.activeRequests
     const req = this.replicator.addUpgrade(activeRequests)
 
-    // TODO: if snapshot, also update the length/byteLength to latest
-    return req.promise
+    if (!this.snapshotted) return req.promise
+    if (!(await req.promise)) return false
+
+    return this._updateSnapshot()
   }
 
   async seek (bytes, opts) {
@@ -476,7 +542,7 @@ module.exports = class Hypercore extends EventEmitter {
     const offset = await s.update()
     if (offset) return offset
 
-    if (this.closing !== null) throw new Error('Session is closed')
+    if (this.closing !== null) throw SESSION_CLOSED()
 
     const activeRequests = (opts && opts.activeRequests) || this.activeRequests
     const req = this.replicator.addSeek(activeRequests, s)
@@ -492,7 +558,8 @@ module.exports = class Hypercore extends EventEmitter {
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
-    if (this.closing !== null) throw new Error('Session is closed')
+    if (this.closing !== null) throw SESSION_CLOSED()
+    if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
 
     const c = this.cache && this.cache.get(index)
     if (c) return c
@@ -567,7 +634,7 @@ module.exports = class Hypercore extends EventEmitter {
 
   async truncate (newLength = 0, fork = -1) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw new Error('Core is not writable')
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
     if (fork === -1) fork = this.core.tree.fork + 1
     await this.core.truncate(newLength, fork, this.auth)
@@ -578,7 +645,7 @@ module.exports = class Hypercore extends EventEmitter {
 
   async append (blocks) {
     if (this.opened === false) await this.opening
-    if (this.writable === false) throw new Error('Core is not writable')
+    if (this.writable === false) throw SESSION_NOT_WRITABLE()
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 

package/lib/core.js

@@ -5,6 +5,7 @@ const Mutex = require('./mutex')
 const MerkleTree = require('./merkle-tree')
 const BlockStore = require('./block-store')
 const Bitfield = require('./bitfield')
+const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = require('./errors')
 const m = require('./messages')
 
 module.exports = class Core {
@@ -52,7 +53,9 @@ module.exports = class Core {
   }
 
   static createAuth (crypto, { publicKey, secretKey }, opts = {}) {
-    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) throw new Error('Invalid key pair')
+    if (secretKey && !crypto.validateKeyPair({ publicKey, secretKey })) {
+      throw BAD_ARGUMENT('Invalid key pair')
+    }
 
     const sign = opts.sign
       ? opts.sign
@@ -88,7 +91,7 @@ module.exports = class Core {
 
     if (!header || overwrite) {
       if (!createIfMissing) {
-        throw new Error('No hypercore is stored here')
+        throw STORAGE_EMPTY('No Hypercore is stored here')
       }
 
       header = {
@@ -110,7 +113,7 @@ module.exports = class Core {
     }
 
     if (opts.keyPair && !b4a.equals(header.signer.publicKey, opts.keyPair.publicKey)) {
-      throw new Error('Another hypercore is stored here')
+      throw STORAGE_CONFLICT('Another Hypercore is stored here')
     }
 
     const tree = await MerkleTree.open(treeFile, { crypto, ...header.tree })
@@ -290,7 +293,7 @@ module.exports = class Core {
     // TODO: move this to tree.js
     const hash = batch.hash()
     if (!batch.signature || !this._signed(batch, hash)) {
-      throw new Error('Remote signature does not match')
+      throw INVALID_SIGNATURE('Proof contains an invalid signature')
     }
 
     await this._mutex.lock()

package/lib/errors.js

@@ -0,0 +1,42 @@
+module.exports = class HypercoreError extends Error {
+  constructor (msg, code) {
+    super(msg)
+    this.code = code
+  }
+
+  static BAD_ARGUMENT (msg) {
+    return new HypercoreError(msg, 'BAD_ARGUMENT')
+  }
+
+  static STORAGE_EMPTY (msg) {
+    return new HypercoreError(msg, 'STORAGE_EMPTY')
+  }
+
+  static STORAGE_CONFLICT (msg) {
+    return new HypercoreError(msg, 'STORAGE_CONFLICT')
+  }
+
+  static INVALID_SIGNATURE (msg) {
+    return new HypercoreError(msg, 'INVALID_SIGNATURE')
+  }
+
+  static INVALID_CAPABILITY (msg) {
+    return new HypercoreError(msg, 'INVALID_CAPABILITY')
+  }
+
+  static SNAPSHOT_NOT_AVAILABLE (msg = 'Snapshot is not available') {
+    return new HypercoreError(msg, 'SNAPSHOT_NOT_AVAILABLE')
+  }
+
+  static REQUEST_CANCELLED (msg = 'Request was cancelled') {
+    return new HypercoreError(msg, 'REQUEST_CANCELLED')
+  }
+
+  static SESSION_NOT_WRITABLE (msg = 'Session is not writable') {
+    return new HypercoreError(msg, 'SESSION_NOT_WRITABLE')
+  }
+
+  static SESSION_CLOSED (msg = 'Session is closed') {
+    return new HypercoreError(msg, 'SESSION_CLOSED')
+  }
+}

package/lib/merkle-tree.js

@@ -228,6 +228,7 @@ class ReorgBatch extends MerkleTreeBatch {
 
     let diff = null
     const ite = flat.iterator(this.diff.index)
+    const startingDiff = this.diff
 
     while ((ite.index & 1) !== 0) {
       const left = n.get(ite.leftChild())
@@ -243,6 +244,7 @@ class ReorgBatch extends MerkleTreeBatch {
 
     if ((this.diff.index & 1) === 0) return true
     if (diff === null) return false
+    if (startingDiff !== this.diff) return false
 
     return this._updateDiffRoot(diff)
   }
@@ -255,10 +257,6 @@ class ReorgBatch extends MerkleTreeBatch {
     const end = Math.min(this.treeLength, spans[1] / 2 + 1)
     const len = end - start
 
-    if (this.diff !== null && len >= this.want.end - this.want.start) {
-      return false
-    }
-
     this.ancestors = start
     this.diff = diff
 

package/lib/replicator.js

@@ -2,6 +2,7 @@ const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
 const RemoteBitfield = require('./remote-bitfield')
+const { REQUEST_CANCELLED, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('./errors')
 const m = require('./messages')
 const caps = require('./caps')
 
@@ -19,6 +20,7 @@ class Attachable {
       session,
       sindex: 0,
       rindex: 0,
+      snapshot: true,
       resolve: null,
       reject: null,
       promise: null
@@ -34,11 +36,11 @@ class Attachable {
     return r
   }
 
-  detach (r) {
+  detach (r, err = null) {
     if (r.context !== this) return false
 
     this._detach(r)
-    this._cancel(r)
+    this._cancel(r, err)
     this.gc()
 
     return true
@@ -60,8 +62,8 @@ class Attachable {
     if (this.refs.length === 0) this._unref()
   }
 
-  _cancel (r) {
-    r.reject(new Error('Request cancelled'))
+  _cancel (r, err) {
+    r.reject(err || REQUEST_CANCELLED())
   }
 
   _unref () {
@@ -84,10 +86,9 @@ class Attachable {
 }
 
 class BlockRequest extends Attachable {
-  constructor (tracker, fork, index) {
+  constructor (tracker, index) {
     super()
 
-    this.fork = fork
     this.index = index
     this.inflight = []
     this.queued = false
@@ -96,15 +97,14 @@ class BlockRequest extends Attachable {
 
   _unref () {
     if (this.inflight.length > 0) return
-    this.tracker.remove(this.fork, this.index)
+    this.tracker.remove(this.index)
   }
 }
 
 class RangeRequest extends Attachable {
-  constructor (ranges, fork, start, end, linear, blocks) {
+  constructor (ranges, start, end, linear, blocks) {
     super()
 
-    this.fork = fork
     this.start = start
     this.end = end
     this.linear = linear
@@ -149,10 +149,9 @@ class UpgradeRequest extends Attachable {
 }
 
 class SeekRequest extends Attachable {
-  constructor (seeks, fork, seeker) {
+  constructor (seeks, seeker) {
     super()
 
-    this.fork = fork
     this.seeker = seeker
     this.inflight = []
     this.seeks = seeks
@@ -200,87 +199,40 @@ class InflightTracker {
 }
 
 class BlockTracker {
-  constructor (core) {
-    this._core = core
-    this._fork = core.tree.fork
-
-    this._indexed = new Map()
-    this._additional = []
+  constructor () {
+    this._map = new Map()
   }
 
-  * [Symbol.iterator] () {
-    yield * this._indexed.values()
-    yield * this._additional
+  [Symbol.iterator] () {
+    return this._map.values()
   }
 
   isEmpty () {
-    return this._indexed.size === 0 && this._additional.length === 0
+    return this._map.size === 0
   }
 
-  has (fork, index) {
-    return this.get(fork, index) !== null
+  has (index) {
+    return this._map.has(index)
   }
 
-  get (fork, index) {
-    if (this._fork === fork) return this._indexed.get(index) || null
-    for (const b of this._additional) {
-      if (b.index === index && b.fork === fork) return b
-    }
-    return null
+  get (index) {
+    return this._map.get(index) || null
   }
 
-  add (fork, index) {
-    // TODO: just rely on someone calling .update(fork) instead
-    if (this._fork !== this._core.tree.fork) this.update(this._core.tree.fork)
-
-    let b = this.get(fork, index)
+  add (index) {
+    let b = this._map.get(index)
     if (b) return b
 
-    b = new BlockRequest(this, fork, index)
-
-    if (fork === this._fork) this._indexed.set(index, b)
-    else this._additional.push(b)
+    b = new BlockRequest(this, index)
+    this._map.set(index, b)
 
     return b
   }
 
-  remove (fork, index) {
-    if (this._fork === fork) {
-      const b = this._indexed.get(index)
-      this._indexed.delete(index)
-      return b || null
-    }
-
-    for (let i = 0; i < this._additional.length; i++) {
-      const b = this._additional[i]
-      if (b.index !== index || b.fork !== fork) continue
-      if (i === this._additional.length - 1) this._additional.pop()
-      else this._additional[i] = this._additional.pop()
-      return b
-    }
-
-    return null
-  }
-
-  update (fork) {
-    if (this._fork === fork) return
-
-    const additional = this._additional
-    this._additional = []
-
-    for (const b of this._indexed.values()) {
-      // TODO: this is only needed cause we hot patch the fork ids below, revert that later
-      if (b.fork !== this._fork) additional.push(b)
-      else this._additional.push(b)
-    }
-    this._indexed.clear()
-
-    for (const b of additional) {
-      if (b.fork === fork) this._indexed.set(b.index, b)
-      else this._additional.push(b)
-    }
-
-    this._fork = fork
+  remove (index) {
+    const b = this.get(index)
+    this._map.delete(index)
+    return b
   }
 }
 
@@ -290,6 +242,7 @@ class Peer {
     this.replicator = replicator
     this.stream = protomux.stream
     this.protomux = protomux
+    this.remotePublicKey = this.stream.remotePublicKey
 
     this.channel = channel
     this.channel.userData = this
@@ -387,7 +340,7 @@ class Peer {
     const expected = caps.replicate(this.stream.isInitiator === false, this.replicator.key, this.stream.handshakeHash)
 
     if (b4a.equals(capability, expected) !== true) { // TODO: change this to a rejection instead, less leakage
-      throw new Error('Remote sent an invalid capability')
+      throw INVALID_CAPABILITY('Remote sent an invalid replication capability')
     }
 
     if (this.remoteOpened === true) return
@@ -425,7 +378,7 @@ class Peer {
 
   async onsync ({ fork, length, remoteLength, canUpgrade, uploading, downloading }) {
     const lengthChanged = length !== this.remoteLength
-    const sameFork = (fork === this.core.tree.fork)
+    const sameFork = fork === this.core.tree.fork
 
     this.remoteSynced = true
     this.remoteFork = fork
@@ -468,12 +421,11 @@ class Peer {
   }
 
   async _updateCanUpgradeAndSync () {
-    const len = this.core.tree.length
-    const fork = this.core.tree.fork
+    const { length, fork } = this.core.tree
 
     const canUpgrade = await this._canUpgrade(this.remoteLength, this.remoteFork)
 
-    if (this.syncsProcessing > 0 || len !== this.core.tree.length || fork !== this.core.tree.fork) {
+    if (this.syncsProcessing > 0 || length !== this.core.tree.length || fork !== this.core.tree.fork) {
       return
     }
     if (canUpgrade === this.canUpgrade) {
@@ -632,19 +584,19 @@ class Peer {
     this.replicator.updatePeer(this)
   }
 
-  _makeRequest (fork, needsUpgrade) {
+  _makeRequest (needsUpgrade) {
     if (needsUpgrade === true && this.replicator._shouldUpgrade(this) === false) {
       return null
     }
 
-    if (needsUpgrade === false && fork === this.core.tree.fork && this.replicator._autoUpgrade(this) === true) {
+    if (needsUpgrade === false && this.replicator._autoUpgrade(this) === true) {
       needsUpgrade = true
     }
 
     return {
       peer: this,
       id: 0,
-      fork,
+      fork: this.remoteFork,
       block: null,
       hash: null,
       seek: null,
@@ -655,7 +607,7 @@ class Peer {
   }
 
   _requestUpgrade (u) {
-    const req = this._makeRequest(u.fork, true)
+    const req = this._makeRequest(true)
     if (req === null) return false
 
     this._send(req)
@@ -664,8 +616,12 @@ class Peer {
   }
 
   _requestSeek (s) {
-    if (s.seeker.start >= this.core.tree.length) {
-      const req = this._makeRequest(s.fork, true)
+    const { length, fork } = this.core.tree
+
+    if (fork !== this.remoteFork) return false
+
+    if (s.seeker.start >= length) {
+      const req = this._makeRequest(true)
 
       // We need an upgrade for the seek, if non can be provided, skip
       if (req === null) return false
@@ -689,14 +645,14 @@ class Peer {
       if (this.core.bitfield.get(index) === true) continue
 
       // Check if this block is currently inflight - if so pick another
-      const b = this.replicator._blocks.get(s.fork, index)
+      const b = this.replicator._blocks.get(index)
       if (b !== null && b.inflight.length > 0) continue
 
       // Block is not inflight, but we only want the hash, check if that is inflight
-      const h = this.replicator._hashes.add(s.fork, index)
+      const h = this.replicator._hashes.add(index)
       if (h.inflight.length > 0) continue
 
-      const req = this._makeRequest(s.fork, false)
+      const req = this._makeRequest(false)
 
       req.hash = { index: 2 * index, nodes: 0 }
       req.seek = { bytes: s.seeker.bytes }
@@ -717,9 +673,11 @@ class Peer {
   }
 
   _requestBlock (b) {
-    if (this.remoteBitfield.get(b.index) === false) return false
+    const { length, fork } = this.core.tree
+
+    if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) return false
 
-    const req = this._makeRequest(b.fork, b.index >= this.core.tree.length)
+    const req = this._makeRequest(b.index >= length)
     if (req === null) return false
 
     req.block = { index: b.index, nodes: 0 }
@@ -731,8 +689,10 @@ class Peer {
   }
 
   _requestRange (r) {
+    const { length, fork } = this.core.tree
+
     const end = Math.min(r.end === -1 ? this.remoteLength : r.end, this.remoteLength)
-    if (end < r.start || r.fork !== this.remoteFork) return false
+    if (end < r.start || fork !== this.remoteFork) return false
 
     const len = end - r.start
     const off = r.start + (r.linear ? 0 : Math.floor(Math.random() * len))
@@ -750,10 +710,10 @@ class Peer {
       if (this.remoteBitfield.get(index) === false) continue
       if (this.core.bitfield.get(index) === true) continue
 
-      const b = this.replicator._blocks.add(r.fork, index)
+      const b = this.replicator._blocks.add(index)
       if (b.inflight.length > 0) continue
 
-      const req = this._makeRequest(r.fork, index >= this.core.tree.length)
+      const req = this._makeRequest(index >= length)
 
       // If the request cannot be satisfied, dealloc the block request if no one is subscribed to it
       if (req === null) {
@@ -773,7 +733,7 @@ class Peer {
   }
 
   _requestForkProof (f) {
-    const req = this._makeRequest(f.fork, false)
+    const req = this._makeRequest(false)
 
     req.upgrade = { start: 0, length: this.remoteLength }
 
@@ -796,7 +756,7 @@ class Peer {
 
       if (this.remoteBitfield.get(index) === false) continue
 
-      const req = this._makeRequest(f.fork, false)
+      const req = this._makeRequest(false)
 
       req.hash = { index: 2 * index, nodes: f.batch.want.nodes }
 
@@ -821,8 +781,12 @@ class Peer {
     }
 
     try {
-      if (req.block !== null && req.fork === fork) req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index)
-      if (req.hash !== null && req.fork === fork) req.hash.nodes = await this.core.tree.missingNodes(req.hash.index)
+      if (req.block !== null && req.fork === fork) {
+        req.block.nodes = await this.core.tree.missingNodes(2 * req.block.index)
+      }
+      if (req.hash !== null && req.fork === fork) {
+        req.hash.nodes = await this.core.tree.missingNodes(req.hash.index)
+      }
     } catch (err) {
       this.stream.destroy(err)
       return
@@ -845,8 +809,8 @@ module.exports = class Replicator {
     this.findingPeers = 0 // updateable from the outside
 
     this._inflight = new InflightTracker()
-    this._blocks = new BlockTracker(core)
-    this._hashes = new BlockTracker(core)
+    this._blocks = new BlockTracker()
+    this._hashes = new BlockTracker()
 
     this._queued = []
 
@@ -869,14 +833,34 @@ module.exports = class Replicator {
     for (const peer of this.peers) peer.protomux.uncork()
   }
 
-  broadcastRange (start, length, drop = false) {
+  // Called externally when a range of new blocks has been processed/removed
+  onhave (start, length, drop = false) {
     for (const peer of this.peers) peer.broadcastRange(start, length, drop)
   }
 
-  localUpgrade () {
+  // Called externally when a truncation upgrade has been processed
+  ontruncate (newLength) {
+    const notify = []
+
+    for (const blk of this._blocks) {
+      if (blk.index < newLength) continue
+      notify.push(blk)
+    }
+
+    for (const blk of notify) {
+      for (const r of blk.refs) {
+        if (r.snapshot === false) continue
+        blk.detach(r, SNAPSHOT_NOT_AVAILABLE())
+      }
+    }
+  }
+
+  // Called externally when a upgrade has been processed
+  onupgrade () {
     for (const peer of this.peers) peer.signalUpgrade()
     if (this._blocks.isEmpty() === false) this._resolveBlocksLocally()
     if (this._upgrade !== null) this._resolveUpgradeRequest(null)
+    if (this._ranges.length !== 0 || this._seeks.length !== 0) this._updateNonPrimary()
   }
 
   addUpgrade (session) {
@@ -901,8 +885,8 @@ module.exports = class Replicator {
     return ref
   }
 
-  addBlock (session, index, fork = this.core.tree.fork) {
-    const b = this._blocks.add(fork, index)
+  addBlock (session, index) {
+    const b = this._blocks.add(index)
     const ref = b.attach(session)
 
     this._queueBlock(b)
@@ -912,7 +896,7 @@ module.exports = class Replicator {
   }
 
   addSeek (session, seeker) {
-    const s = new SeekRequest(this._seeks, this.core.tree.fork, seeker)
+    const s = new SeekRequest(this._seeks, seeker)
     const ref = s.attach(session)
 
     this._seeks.push(s)
@@ -927,7 +911,7 @@ module.exports = class Replicator {
       if (length === -1 || start + length > blocks.length) length = blocks.length - start
     }
 
-    const r = new RangeRequest(this._ranges, this.core.tree.fork, start, length === -1 ? -1 : start + length, linear, blocks)
+    const r = new RangeRequest(this._ranges, start, length === -1 ? -1 : start + length, linear, blocks)
     const ref = r.attach(session)
 
     this._ranges.push(r)
@@ -940,13 +924,13 @@ module.exports = class Replicator {
   }
 
   cancel (ref) {
-    ref.context.detach(ref)
+    ref.context.detach(ref, null)
   }
 
   clearRequests (session) {
     while (session.length > 0) {
       const ref = session[session.length - 1]
-      ref.context.detach(ref)
+      ref.context.detach(ref, null)
     }
 
     this.updateAll()
@@ -1046,7 +1030,7 @@ module.exports = class Replicator {
   }
 
   _autoUpgrade (peer) {
-    return this._upgrade !== null && this._shouldUpgrade(peer)
+    return this._upgrade !== null && peer.remoteFork === this.core.tree.fork && this._shouldUpgrade(peer)
   }
 
   _addPeer (peer) {
@@ -1099,12 +1083,12 @@ module.exports = class Replicator {
     // Currently the block tracker does not support deletes during iteration, so we make
     // sure to clear them afterwards.
     for (const b of clear) {
-      this._blocks.remove(b.fork, b.index)
+      this._blocks.remove(b.index)
     }
   }
 
-  _resolveBlockRequest (tracker, fork, index, value, req) {
-    const b = tracker.remove(fork, index)
+  _resolveBlockRequest (tracker, index, value, req) {
+    const b = tracker.remove(index)
     if (b === null) return false
 
     removeInflight(b.inflight, req)
@@ -1130,7 +1114,7 @@ module.exports = class Replicator {
   _clearInflightBlock (tracker, req) {
     const isBlock = tracker === this._blocks
     const index = isBlock === true ? req.block.index : req.hash.index / 2
-    const b = tracker.get(req.fork, index)
+    const b = tracker.get(index)
 
     if (b === null || removeInflight(b.inflight, req) === false) return
 
@@ -1251,11 +1235,11 @@ module.exports = class Replicator {
 
   _ondata (peer, req, data) {
     if (data.block !== null) {
-      this._resolveBlockRequest(this._blocks, data.fork, data.block.index, data.block.value, req)
+      this._resolveBlockRequest(this._blocks, data.block.index, data.block.value, req)
     }
 
     if (data.hash !== null && (data.hash.index & 1) === 0) {
-      this._resolveBlockRequest(this._hashes, data.fork, data.hash.index / 2, null, req)
+      this._resolveBlockRequest(this._hashes, data.hash.index / 2, null, req)
     }
 
     if (this._upgrade !== null) {
@@ -1286,14 +1270,14 @@ module.exports = class Replicator {
 
     if (f.batch) {
       await f.batch.update(data)
-    } else {
+    } else if (data.upgrade) {
       f.batch = await this.core.tree.reorg(data)
 
       // Remove "older" reorgs in progress as we just verified this one.
       this._clearOldReorgs(f.fork)
     }
 
-    if (f.batch.finished) {
+    if (f.batch && f.batch.finished) {
       if (this._addUpgradeMaybe() !== null) {
         await this._applyReorg(f)
       }
@@ -1302,6 +1286,7 @@ module.exports = class Replicator {
     this.updateAll()
   }
 
+  // Never throws, allowed to run in the background
   async _applyReorg (f) {
     // TODO: more optimal here to check if potentially a better reorg
     // is available, ie higher fork, and request that one first.
@@ -1330,13 +1315,10 @@ module.exports = class Replicator {
 
     // TODO: all the remaining is a tmp workaround until we have a flag/way for ANY_FORK
     for (const r of this._ranges) {
-      r.fork = this.core.tree.fork
       r.start = r.userStart
       r.end = r.userEnd
     }
-    for (const s of this._seeks) s.fork = this.core.tree.fork
-    for (const b of this._blocks) b.fork = this.core.tree.fork
-    this._blocks.update(this.core.tree.fork)
+
     this.updateAll()
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.35",
+  "version": "10.0.0-alpha.38",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -53,7 +53,7 @@
   "devDependencies": {
     "brittle": "^2.0.0",
     "hyperswarm": "next",
-    "random-access-memory": "^3.1.2",
+    "random-access-memory": "^4.1.0",
     "standard": "^16.0.3",
     "tmp-promise": "^3.0.2"
   },