hypercore 10.0.0-alpha.38 → 10.0.0-alpha.40

package/README.md

@@ -245,6 +245,12 @@ How much data is available on this core in bytes?
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
+#### `core.contiguousLength`
+
+How many blocks are contiguously available starting from the first block of this core?
+
+Populated after `ready` has been emitted. Will be `0` before the event.
+
 #### `core.fork`
 
 What is the current fork id of this core?

package/index.js

@@ -190,6 +190,18 @@ module.exports = class Hypercore extends EventEmitter {
     })
 
     s._passCapabilities(this)
+
+    if (opts.encryptionKey) {
+      // Only override the block encryption if its either not already set or if
+      // the caller provided a different key.
+      if (
+        !this.encryption ||
+        !b4a.equals(this.encryption.key, opts.encryptionKey)
+      ) {
+        this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
+      }
+    }
+
     this.sessions.push(s)
 
     return s
@@ -368,24 +380,34 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   replicate (isInitiator, opts = {}) {
+    // Only limitation here is that ondiscoverykey doesn't work atm when passing a muxer directly,
+    // because it doesn't really make a lot of sense.
+    if (Protomux.isProtomux(isInitiator)) return this._attachToMuxer(isInitiator, opts)
+
     const protocolStream = Hypercore.createProtocolStream(isInitiator, opts)
     const noiseStream = protocolStream.noiseStream
     const protocol = noiseStream.userData
 
+    this._attachToMuxer(protocol, opts)
+
+    return protocolStream
+  }
+
+  _attachToMuxer (mux, opts) {
     // If the user wants to, we can make this replication run in a session
     // that way the core wont close "under them" during replication
     if (opts.session) {
       const s = this.session()
-      protocolStream.on('close', () => s.close().catch(noop))
+      mux.stream.on('close', () => s.close().catch(noop))
     }
 
     if (this.opened) {
-      this.replicator.attachTo(protocol)
+      this.replicator.attachTo(mux)
     } else {
-      this.opening.then(() => this.replicator.attachTo(protocol), protocol.destroy.bind(protocol))
+      this.opening.then(() => this.replicator.attachTo(mux), mux.destroy.bind(mux))
     }
 
-    return protocolStream
+    return mux
   }
 
   get discoveryKey () {
@@ -404,6 +426,10 @@ module.exports = class Hypercore extends EventEmitter {
       : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
   }
 
+  get contiguousLength () {
+    return this.core === null ? 0 : this.core.header.contiguousLength
+  }
+
   get fork () {
     return this.core === null ? 0 : this.core.tree.fork
   }

package/lib/core.js

@@ -26,6 +26,8 @@ module.exports = class Core {
     this._verifiesFlushed = null
     this._mutex = new Mutex()
     this._legacy = legacy
+
+    this._updateContiguousLength(header.contiguousLength)
   }
 
   static async open (storage, opts = {}) {
@@ -106,7 +108,8 @@ module.exports = class Core {
         signer: opts.keyPair || crypto.keyPair(),
         hints: {
           reorgs: []
-        }
+        },
+        contiguousLength: 0
       }
 
       await oplog.flush(header)
@@ -192,6 +195,16 @@ module.exports = class Core {
     await this.blocks.put(index, value, byteOffset)
   }
 
+  _updateContiguousLength (index, length = 0) {
+    if (index === this.header.contiguousLength) {
+      let i = this.header.contiguousLength + length
+
+      while (this.bitfield.get(i)) i++
+
+      this.header.contiguousLength = i
+    }
+  }
+
   async userData (key, value) {
     // TODO: each oplog append can set user data, so we should have a way
     // to just hitch a ride on one of the other ongoing appends?
@@ -271,6 +284,7 @@ module.exports = class Core {
       this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true)
       batch.commit()
 
+      this.header.contiguousLength = batch.length
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
@@ -312,7 +326,11 @@ module.exports = class Core {
 
       await this.oplog.append([entry], false)
 
-      if (bitfield) this.bitfield.set(bitfield.start, true)
+      if (bitfield) {
+        this.bitfield.set(bitfield.start, true)
+        this._updateContiguousLength(bitfield.start, bitfield.length)
+      }
+
       batch.commit()
 
       this.header.tree.fork = batch.fork
@@ -366,8 +384,13 @@ module.exports = class Core {
           continue
         }
 
-        if (bitfield) this.bitfield.set(bitfield.start, true)
+        if (bitfield) {
+          this.bitfield.set(bitfield.start, true)
+          this._updateContiguousLength(bitfield.start, bitfield.length)
+        }
+
         batch.commit()
+
         this.onupdate(0, bitfield, value, from)
       }
 
@@ -448,6 +471,7 @@ module.exports = class Core {
 
     const appended = batch.length > batch.ancestors
 
+    this.header.contiguousLength = Math.min(batch.ancestors, this.header.contiguousLength)
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()

package/lib/messages.js

@@ -622,6 +622,7 @@ oplog.header = {
     treeHeader.preencode(state, h.tree)
     keyPair.preencode(state, h.signer)
     hints.preencode(state, h.hints)
+    c.uint.preencode(state, h.contiguousLength)
   },
   encode (state, h) {
     state.buffer[state.start++] = 0 // version
@@ -630,6 +631,7 @@ oplog.header = {
     treeHeader.encode(state, h.tree)
     keyPair.encode(state, h.signer)
     hints.encode(state, h.hints)
+    c.uint.encode(state, h.contiguousLength)
   },
   decode (state) {
     const version = c.uint.decode(state)
@@ -643,7 +645,8 @@ oplog.header = {
       userData: keyValueArray.decode(state),
       tree: treeHeader.decode(state),
       signer: keyPair.decode(state),
-      hints: hints.decode(state)
+      hints: hints.decode(state),
+      contiguousLength: state.end > state.start ? c.uint.decode(state) : 0
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.38",
+  "version": "10.0.0-alpha.40",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {
@@ -54,6 +54,7 @@
     "brittle": "^2.0.0",
     "hyperswarm": "next",
     "random-access-memory": "^4.1.0",
+    "random-access-memory-overlay": "^1.0.0",
     "standard": "^16.0.3",
     "tmp-promise": "^3.0.2"
   },

package/CHANGELOG.md

@@ -1,37 +0,0 @@
-# CHANGELOG
-
-See [UPGRADE.md](UPGRADE.md) for notes on breaking changes for downstream developers.
-
-## Current
-
-## v9.5.0
-
-- Feed close makes the replication detach the channel used for this particular stream.
-
-## v9.4.0
-
-- feed.get accepts onwait hook, that is called if get is waiting for a network peer.
-
-## v9.3.0
-
-- feed.get returns an id, that can be used to cancel a pending get with feed.cancel(getId)
-
-## v9.2.0
-
-- Add `maxBlockSize` on write streams to auto chunk big blocks being written.
-
-## v9.1.0
-
-- Make peer.remoteOpened public ([#268](https://github.com/hypercore-protocol/hypercore/pull/268))
-
-## v9.0.1
-
-- Upgraded standard to v14 with subsequent formatting tweaks
-- createReadStream is up to 8x faster now! ([#261](https://github.com/hypercore-protocol/hypercore/pull/261) by [@tinchoz49](https://github.com/tinchoz49))
-- Fixed benchmarks ([#266](https://github.com/hypercore-protocol/hypercore/pull/266) by [@fwip](https://github.com/fwip))
-
-## v9.0.0
-
-- Ease of use update to signatures, https://github.com/mafintosh/hypercore/issues/260
-- Updates [noise-protocol](https://github.com/emilbayes/noise-protocol) to latest, which uses chacha instead of xchacha and moves avoid from the sodium kx api for better compatability with the rest of the Noise ecosystem.
-- Updates [sodium-native](https://github.com/sodium-friends/sodium-native) from 2 to 3 across the board. 3 uses n-api, so no more builds needed when Node is released.