hypercore 10.0.0-alpha.42 → 10.0.0-alpha.45

package/README.md

@@ -62,6 +62,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 {
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
+  sparse: true, // enable sparse mode, counting unavailable blocks towards core.length and core.byteLength
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
   encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
@@ -235,13 +236,13 @@ Buffer containing the optional block encryption key of this core. Will be `null`
 
 #### `core.length`
 
-How many blocks of data are available on this core?
+How many blocks of data are available on this core? If `sparse: false`, this will equal `core.contiguousLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
 #### `core.byteLength`
 
-How much data is available on this core in bytes?
+How much data is available on this core in bytes? If `sparse: false`, this will equal `core.contiguousByteLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
@@ -251,6 +252,12 @@ How many blocks are contiguously available starting from the first block of this
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
+#### `core.contiguousByteLength`
+
+How much data is contiguously available starting from the first block of this core?
+
+Populated after `ready` has been emitted. Will be `0` before the event.
+
 #### `core.fork`
 
 What is the current fork id of this core?

package/index.js

@@ -66,6 +66,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opened = false
     this.closed = false
     this.snapshotted = !!opts.snapshot
+    this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
@@ -151,7 +152,6 @@ module.exports = class Hypercore extends EventEmitter {
       }
       if (opts.keepAlive !== false) {
         noiseStream.setKeepAlive(5000)
-        noiseStream.setTimeout(10000)
       }
       noiseStream.userData = protocol
     }
@@ -191,8 +191,10 @@ module.exports = class Hypercore extends EventEmitter {
 
     s._passCapabilities(this)
 
-    // Pass on the cache unless explicitly disabled.
-    if (opts.cache !== false) s.cache = this.cache
+    // Configure the cache unless explicitly disabled.
+    if (opts.cache !== false) {
+      s.cache = opts.cache === true || !opts.cache ? this.cache : opts.cache
+    }
 
     ensureEncryption(s, opts)
 
@@ -271,6 +273,9 @@ module.exports = class Hypercore extends EventEmitter {
       this.encodeBatch = opts.encodeBatch
     }
 
+    // Start continous replication if not in sparse mode.
+    if (!this.sparse) this.download({ start: 0, end: -1 })
+
     // This is a hidden option that's only used by Corestore.
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
     if (opts._preready) await opts._preready(this)
@@ -292,7 +297,8 @@ module.exports = class Hypercore extends EventEmitter {
       crypto: this.crypto,
       legacy: opts.legacy,
       auth: opts.auth,
-      onupdate: this._oncoreupdate.bind(this)
+      onupdate: this._oncoreupdate.bind(this),
+      oncontigupdate: this._oncorecontigupdate.bind(this)
     })
 
     if (opts.userData) {
@@ -411,21 +417,27 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get length () {
-    return this._snapshot
-      ? this._snapshot.length
-      : (this.core === null ? 0 : this.core.tree.length)
+    if (this._snapshot) return this._snapshot.length
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousLength
+    return this.core.tree.length
   }
 
   get byteLength () {
-    return this._snapshot
-      ? this._snapshot.byteLength
-      : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
+    if (this._snapshot) return this._snapshot.byteLength
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousByteLength
+    return this.core.tree.byteLength - (this.core.tree.length * this.padding)
   }
 
   get contiguousLength () {
     return this.core === null ? 0 : this.core.header.contiguousLength
   }
 
+  get contiguousByteLength () {
+    return 0
+  }
+
   get fork () {
     return this.core === null ? 0 : this.core.tree.fork
   }
@@ -473,9 +485,10 @@ module.exports = class Hypercore extends EventEmitter {
           if (s._snapshot && bitfield.start < s._snapshot.compatLength) s._snapshot.compatLength = bitfield.start
         }
 
-        if (appended) {
-          s.emit('append')
-        }
+        // For sparse sessions, immediately emit appends. Non-sparse sessions
+        // are handled separately and only emit appends when their contiguous
+        // length is updated.
+        if (appended && s.sparse) s.emit('append')
       }
 
       this.replicator.onupgrade()
@@ -494,6 +507,16 @@ module.exports = class Hypercore extends EventEmitter {
     }
   }
 
+  _oncorecontigupdate () {
+    // For non-sparse sessions, emit appends only when the contiguous length is
+    // updated.
+    for (let i = 0; i < this.sessions.length; i++) {
+      const s = this.sessions[i]
+
+      if (!s.sparse) s.emit('append')
+    }
+  }
+
   _onpeerupdate (added, peer) {
     const name = added ? 'peer-add' : 'peer-remove'
 

package/lib/core.js

@@ -9,8 +9,9 @@ const { BAD_ARGUMENT, STORAGE_EMPTY, STORAGE_CONFLICT, INVALID_SIGNATURE } = req
 const m = require('./messages')
 
 module.exports = class Core {
-  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate) {
+  constructor (header, crypto, oplog, tree, blocks, bitfield, auth, legacy, onupdate, oncontigupdate) {
     this.onupdate = onupdate
+    this.oncontigupdate = oncontigupdate
     this.header = header
     this.crypto = crypto
     this.oplog = oplog
@@ -164,7 +165,7 @@ module.exports = class Core {
       }
     }
 
-    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop)
+    return new this(header, crypto, oplog, tree, blocks, bitfield, auth, !!opts.legacy, opts.onupdate || noop, opts.oncontigupdate || noop)
   }
 
   _shouldFlush () {
@@ -202,6 +203,7 @@ module.exports = class Core {
       while (this.bitfield.get(i)) i++
 
       this.header.contiguousLength = i
+      this.oncontigupdate()
     }
   }
 
@@ -285,6 +287,7 @@ module.exports = class Core {
       batch.commit()
 
       this.header.contiguousLength = batch.length
+      this.oncontigupdate()
       this.header.tree.length = batch.length
       this.header.tree.rootHash = hash
       this.header.tree.signature = batch.signature
@@ -472,6 +475,7 @@ module.exports = class Core {
     const appended = batch.length > batch.ancestors
 
     this.header.contiguousLength = Math.min(batch.ancestors, this.header.contiguousLength)
+    this.oncontigupdate()
     this.header.tree.fork = batch.fork
     this.header.tree.length = batch.length
     this.header.tree.rootHash = batch.hash()

package/lib/replicator.js

@@ -726,6 +726,10 @@ class Peer {
       b.inflight.push(req)
       this._send(req)
 
+      // Don't think this will ever happen, as the pending queue is drained before the range queue
+      // but doesn't hurt to check this explicitly here also.
+      if (b.queued) b.queued = false
+
       return true
     }
 
@@ -906,9 +910,9 @@ module.exports = class Replicator {
   }
 
   addRange (session, { start = 0, end = -1, length = toLength(start, end), blocks = null, linear = false } = {}) {
-    if (blocks !== null) {
-      if (start >= blocks.length) start = blocks.length
-      if (length === -1 || start + length > blocks.length) length = blocks.length - start
+    if (blocks !== null) { // if using blocks, start, end just acts as frames around the blocks array
+      start = 0
+      end = length = blocks.length
     }
 
     const r = new RangeRequest(this._ranges, start, length === -1 ? -1 : start + length, linear, blocks)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.42",
+  "version": "10.0.0-alpha.45",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {