hypercore 10.0.0-alpha.41 → 10.0.0-alpha.44

package/README.md

@@ -62,6 +62,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 {
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
+  sparse: true, // enable sparse mode, counting unavailable blocks towards core.length and core.byteLength
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
   encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
@@ -235,13 +236,13 @@ Buffer containing the optional block encryption key of this core. Will be `null`
 
 #### `core.length`
 
-How many blocks of data are available on this core?
+How many blocks of data are available on this core? If `sparse: false`, this will equal `core.contiguousLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
 #### `core.byteLength`
 
-How much data is available on this core in bytes?
+How much data is available on this core in bytes? If `sparse: false`, this will equal `core.contiguousByteLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
@@ -251,6 +252,12 @@ How many blocks are contiguously available starting from the first block of this
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
+#### `core.contiguousByteLength`
+
+How much data is contiguously available starting from the first block of this core?
+
+Populated after `ready` has been emitted. Will be `0` before the event.
+
 #### `core.fork`
 
 What is the current fork id of this core?

package/index.js

@@ -66,6 +66,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opened = false
     this.closed = false
     this.snapshotted = !!opts.snapshot
+    this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
@@ -151,7 +152,6 @@ module.exports = class Hypercore extends EventEmitter {
       }
       if (opts.keepAlive !== false) {
         noiseStream.setKeepAlive(5000)
-        noiseStream.setTimeout(10000)
       }
       noiseStream.userData = protocol
     }
@@ -191,17 +191,13 @@ module.exports = class Hypercore extends EventEmitter {
 
     s._passCapabilities(this)
 
-    if (opts.encryptionKey) {
-      // Only override the block encryption if its either not already set or if
-      // the caller provided a different key.
-      if (
-        !s.encryption ||
-        !b4a.equals(s.encryption.key, opts.encryptionKey)
-      ) {
-        s.encryption = new BlockEncryption(opts.encryptionKey, s.key)
-      }
+    // Configure the cache unless explicitly disabled.
+    if (opts.cache !== false) {
+      s.cache = opts.cache === true || !opts.cache ? this.cache : opts.cache
     }
 
+    ensureEncryption(s, opts)
+
     this.sessions.push(s)
 
     return s
@@ -229,6 +225,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.storage = from.storage
     this.replicator.findingPeers += this._findingPeers
 
+    ensureEncryption(this, opts)
+
     this.sessions.push(this)
   }
 
@@ -275,6 +273,9 @@ module.exports = class Hypercore extends EventEmitter {
       this.encodeBatch = opts.encodeBatch
     }
 
+    // Start continous replication if not in sparse mode.
+    if (!this.sparse) this.download({ start: 0, end: -1 })
+
     // This is a hidden option that's only used by Corestore.
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
     if (opts._preready) await opts._preready(this)
@@ -415,21 +416,27 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get length () {
-    return this._snapshot
-      ? this._snapshot.length
-      : (this.core === null ? 0 : this.core.tree.length)
+    if (this._snapshot) return this._snapshot.length
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousLength
+    return this.core.tree.length
   }
 
   get byteLength () {
-    return this._snapshot
-      ? this._snapshot.byteLength
-      : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
+    if (this._snapshot) return this._snapshot.byteLength
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousByteLength
+    return this.core.tree.byteLength - (this.core.tree.length * this.padding)
   }
 
   get contiguousLength () {
     return this.core === null ? 0 : this.core.header.contiguousLength
   }
 
+  get contiguousByteLength () {
+    return 0
+  }
+
   get fork () {
     return this.core === null ? 0 : this.core.tree.fork
   }
@@ -587,33 +594,57 @@ module.exports = class Hypercore extends EventEmitter {
     if (this.closing !== null) throw SESSION_CLOSED()
     if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
 
-    const c = this.cache && this.cache.get(index)
-    if (c) return c
-    const fork = this.core.tree.fork
-    const b = await this._get(index, opts)
-    if (this.cache && fork === this.core.tree.fork && b) this.cache.set(index, b)
-    return b
+    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
+
+    let req = this.cache && this.cache.get(index)
+    if (!req) req = this._get(index, opts)
+
+    let block = await req
+    if (!block) return null
+
+    if (this.encryption) {
+      // Copy the block as it might be shared with other sessions.
+      block = b4a.from(block)
+
+      this.encryption.decrypt(index, block)
+    }
+
+    return this._decode(encoding, block)
   }
 
   async _get (index, opts) {
-    const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
-
-    let block
+    let req
 
     if (this.core.bitfield.get(index)) {
-      block = await this.core.blocks.get(index)
+      req = this.core.blocks.get(index)
+
+      if (this.cache) this.cache.set(index, req)
     } else {
       if (opts && opts.wait === false) return null
       if (opts && opts.onwait) opts.onwait(index)
 
       const activeRequests = (opts && opts.activeRequests) || this.activeRequests
-      const req = this.replicator.addBlock(activeRequests, index)
 
-      block = await req.promise
+      req = this._cacheOnResolve(
+        index,
+        this.replicator
+          .addBlock(activeRequests, index)
+          .promise,
+        this.core.tree.fork
+      )
     }
 
-    if (this.encryption) this.encryption.decrypt(index, block)
-    return this._decode(encoding, block)
+    return req
+  }
+
+  async _cacheOnResolve (index, req, fork) {
+    const block = await req
+
+    if (this.cache && fork === this.core.tree.fork) {
+      this.cache.set(index, Promise.resolve(block))
+    }
+
+    return block
   }
 
   createReadStream (opts) {
@@ -765,7 +796,7 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   _decode (enc, block) {
-    block = block.subarray(this.padding)
+    if (this.padding) block = block.subarray(this.padding)
     if (enc) return c.decode(enc, block)
     return block
   }
@@ -797,3 +828,11 @@ function preappend (blocks) {
     this.encryption.encrypt(offset + i, blocks[i], fork)
   }
 }
+
+function ensureEncryption (core, opts) {
+  if (!opts.encryptionKey) return
+  // Only override the block encryption if its either not already set or if
+  // the caller provided a different key.
+  if (core.encryption && b4a.equals(core.encryption.key, opts.encryptionKey)) return
+  core.encryption = new BlockEncryption(opts.encryptionKey, core.key)
+}

package/lib/replicator.js

@@ -726,6 +726,10 @@ class Peer {
       b.inflight.push(req)
       this._send(req)
 
+      // Don't think this will ever happen, as the pending queue is drained before the range queue
+      // but doesn't hurt to check this explicitly here also.
+      if (b.queued) b.queued = false
+
       return true
     }
 
@@ -906,9 +910,9 @@ module.exports = class Replicator {
   }
 
   addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false } = {}) {
-    if (blocks !== null) {
-      if (start >= blocks.length) start = blocks.length
-      if (length === -1 || start + length > blocks.length) length = blocks.length - start
+    if (blocks !== null) { // if using blocks, start, end just acts as frames around the blocks array
+      start = 0
+      end = length = blocks.length
     }
 
     const r = new RangeRequest(this._ranges, start, length === -1 ? -1 : start + length, linear, blocks)
@@ -1054,7 +1058,7 @@ module.exports = class Replicator {
   }
 
   _queueBlock (b) {
-    if (b.queued === true) return
+    if (b.inflight.length > 0 || b.queued === true) return
     b.queued = true
     this._queued.push(b)
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.41",
+  "version": "10.0.0-alpha.44",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {