hypercore 10.0.0-alpha.40 → 10.0.0-alpha.43

package/README.md

@@ -62,6 +62,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 {
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
+  sparse: true, // enable sparse mode, counting unavailable blocks towards core.length and core.byteLength
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
   encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
@@ -235,13 +236,13 @@ Buffer containing the optional block encryption key of this core. Will be `null`
 
 #### `core.length`
 
-How many blocks of data are available on this core?
+How many blocks of data are available on this core? If `sparse: false`, this will equal `core.contiguousLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
 #### `core.byteLength`
 
-How much data is available on this core in bytes?
+How much data is available on this core in bytes? If `sparse: false`, this will equal `core.contiguousByteLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
@@ -251,6 +252,12 @@ How many blocks are contiguously available starting from the first block of this
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
+#### `core.contiguousByteLength`
+
+How much data is contiguously available starting from the first block of this core?
+
+Populated after `ready` has been emitted. Will be `0` before the event.
+
 #### `core.fork`
 
 What is the current fork id of this core?

package/index.js

@@ -66,6 +66,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opened = false
     this.closed = false
     this.snapshotted = !!opts.snapshot
+    this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
@@ -151,7 +152,6 @@ module.exports = class Hypercore extends EventEmitter {
       }
       if (opts.keepAlive !== false) {
         noiseStream.setKeepAlive(5000)
-        noiseStream.setTimeout(10000)
       }
       noiseStream.userData = protocol
     }
@@ -191,16 +191,10 @@ module.exports = class Hypercore extends EventEmitter {
 
     s._passCapabilities(this)
 
-    if (opts.encryptionKey) {
-      // Only override the block encryption if its either not already set or if
-      // the caller provided a different key.
-      if (
-        !this.encryption ||
-        !b4a.equals(this.encryption.key, opts.encryptionKey)
-      ) {
-        this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
-      }
-    }
+    // Pass on the cache unless explicitly disabled.
+    if (opts.cache !== false) s.cache = this.cache
+
+    ensureEncryption(s, opts)
 
     this.sessions.push(s)
 
@@ -229,6 +223,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.storage = from.storage
     this.replicator.findingPeers += this._findingPeers
 
+    ensureEncryption(this, opts)
+
     this.sessions.push(this)
   }
 
@@ -275,6 +271,9 @@ module.exports = class Hypercore extends EventEmitter {
       this.encodeBatch = opts.encodeBatch
     }
 
+    // Start continous replication if not in sparse mode.
+    if (!this.sparse) this.download({ start: 0, end: -1 })
+
     // This is a hidden option that's only used by Corestore.
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
     if (opts._preready) await opts._preready(this)
@@ -415,21 +414,27 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get length () {
-    return this._snapshot
-      ? this._snapshot.length
-      : (this.core === null ? 0 : this.core.tree.length)
+    if (this._snapshot) return this._snapshot.length
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousLength
+    return this.core.tree.length
   }
 
   get byteLength () {
-    return this._snapshot
-      ? this._snapshot.byteLength
-      : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
+    if (this._snapshot) return this._snapshot.byteLength
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousByteLength
+    return this.core.tree.byteLength - (this.core.tree.length * this.padding)
   }
 
   get contiguousLength () {
     return this.core === null ? 0 : this.core.header.contiguousLength
   }
 
+  get contiguousByteLength () {
+    return 0
+  }
+
   get fork () {
     return this.core === null ? 0 : this.core.tree.fork
   }
@@ -587,33 +592,57 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.closing !== null) throw SESSION_CLOSED()
     if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
 
-    const c = this.cache && this.cache.get(index)
-    if (c) return c
-    const fork = this.core.tree.fork
-    const b = await this._get(index, opts)
-    if (this.cache && fork === this.core.tree.fork && b) this.cache.set(index, b)
-    return b
+    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
+
+    let req = this.cache && this.cache.get(index)
+    if (!req) req = this._get(index, opts)
+
+    let block = await req
+    if (!block) return null
+
+    if (this.encryption) {
+      // Copy the block as it might be shared with other sessions.
+      block = b4a.from(block)
+
+      this.encryption.decrypt(index, block)
+    }
+
+    return this._decode(encoding, block)
   }
 
   async _get (index, opts) {
-    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
-
-    let block
+    let req
 
     if (this.core.bitfield.get(index)) {
-      block = await this.core.blocks.get(index)
+      req = this.core.blocks.get(index)
+
+      if (this.cache) this.cache.set(index, req)
     } else {
       if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index)
 
       const activeRequests = (opts && opts.activeRequests) || this.activeRequests
-      const req = this.replicator.addBlock(activeRequests, index)
 
-      block = await req.promise
+      req = this._cacheOnResolve(
+        index,
+        this.replicator
+          .addBlock(activeRequests, index)
+          .promise,
+        this.core.tree.fork
+      )
     }
 
-    if (this.encryption) this.encryption.decrypt(index, block)
-    return this._decode(encoding, block)
+    return req
+  }
+
+  async _cacheOnResolve (index, req, fork) {
+    const block = await req
+
+    if (this.cache && fork === this.core.tree.fork) {
+      this.cache.set(index, Promise.resolve(block))
+    }
+
+    return block
   }
 
   createReadStream (opts) {
@@ -765,7 +794,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   _decode (enc, block) {
-    block = block.subarray(this.padding)
+    if (this.padding) block = block.subarray(this.padding)
     if (enc) return c.decode(enc, block)
     return block
   }
@@ -797,3 +826,11 @@ function preappend (blocks) {
     this.encryption.encrypt(offset + i, blocks[i], fork)
   }
 }
+
+function ensureEncryption (core, opts) {
+  if (!opts.encryptionKey) return
+  // Only override the block encryption if its either not already set or if
+  // the caller provided a different key.
+  if (core.encryption && b4a.equals(core.encryption.key, opts.encryptionKey)) return
+  core.encryption = new BlockEncryption(opts.encryptionKey, core.key)
+}

package/lib/replicator.js

@@ -726,6 +726,10 @@ class Peer {
       b.inflight.push(req)
       this._send(req)
 
+      // Don't think this will ever happen, as the pending queue is drained before the range queue
+      // but doesn't hurt to check this explicitly here also.
+      if (b.queued) b.queued = false
+
       return true
     }
 
@@ -1054,7 +1058,7 @@ module.exports = class Replicator {
   }
 
   _queueBlock (b) {
-    if (b.queued === true) return
+    if (b.inflight.length > 0 || b.queued === true) return
     b.queued = true
     this._queued.push(b)
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.40",
+  "version": "10.0.0-alpha.43",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {