hypercore 10.0.0-alpha.5 → 10.0.0-alpha.6

package/README.md

@@ -113,6 +113,7 @@ A range can have the following properties:
 {
   start: startIndex,
   end: nonInclusiveEndIndex,
+  blocks: [index1, index2, ...],
   linear: false // download range linearly and not randomly
 }
 ```
@@ -125,6 +126,12 @@ To download the full core continously (often referred to as non sparse mode) do
 core.download({ start: 0, end: -1 })
 ```
 
+To downloaded a discrete range of blocks pass a list of indices.
+
+```js
+core.download({ blocks: [4, 9, 7] });
+```
+
 To cancel downloading a range simply destroy the range instance.
 
 ``` js

package/index.js

@@ -28,14 +28,17 @@ module.exports = class Hypercore extends EventEmitter {
       opts = key
       key = null
     }
+
     if (key && typeof key === 'string') {
       key = Buffer.from(key, 'hex')
     }
-    if (key && key.byteLength !== 32) {
+
+    if (!opts) opts = {}
+
+    if (!opts.crypto && key && key.byteLength !== 32) {
       throw new Error('Hypercore key should be 32 bytes')
     }
 
-    if (!opts) opts = {}
     if (!storage) storage = opts.storage
 
     this[promises] = true
@@ -371,13 +374,27 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   download (range) {
-    const start = (range && range.start) || 0
-    const end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
     const linear = !!(range && range.linear)
 
-    // TODO: support range.blocks
+    let start
+    let end
+    let filter
+
+    if (range && range.blocks) {
+      const blocks = range.blocks instanceof Set
+        ? range.blocks
+        : new Set(range.blocks)
+
+      start = range.start || (blocks.size ? min(range.blocks) : 0)
+      end = range.end || (blocks.size ? max(range.blocks) + 1 : 0)
+
+      filter = (i) => blocks.has(i)
+    } else {
+      start = (range && range.start) || 0
+      end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
+    }
 
-    const r = Replicator.createRange(start, end, linear)
+    const r = Replicator.createRange(start, end, filter, linear)
 
     if (this.opened) this.replicator.addRange(r)
     else this.opening.then(() => this.replicator.addRange(r), noop)
@@ -459,3 +476,16 @@ function requireMaybe (name) {
 function toHex (buf) {
   return buf && buf.toString('hex')
 }
+
+function reduce (iter, fn, acc) {
+  for (const item of iter) acc = fn(acc, item)
+  return acc
+}
+
+function min (arr) {
+  return reduce(arr, (a, b) => Math.min(a, b), Infinity)
+}
+
+function max (arr) {
+  return reduce(arr, (a, b) => Math.max(a, b), -Infinity)
+}

package/lib/replicator.js

@@ -120,9 +120,10 @@ class Seek {
 }
 
 class Range {
-  constructor (start, end, linear) {
+  constructor (start, end, filter, linear) {
     this.start = start
     this.end = end
+    this.filter = filter
     this.linear = !!linear
     this.promise = null
     this.done = false
@@ -145,7 +146,7 @@ class Range {
     }
 
     for (; this._start < this.end; this._start++) {
-      if (!bitfield.get(this._start)) return false
+      if (this.filter(this._start) && !bitfield.get(this._start)) return false
     }
 
     return true
@@ -299,10 +300,13 @@ class RequestPool {
     const end = range.end === -1 ? peer.state.length : range.end
     if (end <= range._start) return false
 
-    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0)
+    if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0, range.filter)
 
     const r = range._start + Math.floor(Math.random() * (end - range._start))
-    return !!(this._requestRange(peer, r, end, 0, 0) || this._requestRange(peer, range._start, r, 0, 0))
+    return !!(
+      this._requestRange(peer, r, end, 0, 0, range.filter) ||
+      this._requestRange(peer, range._start, r, 0, 0, range.filter)
+    )
   }
 
   _updateUpgrade (peer) {
@@ -331,7 +335,7 @@ class RequestPool {
     this.pendingUpgrade = null
   }
 
-  _requestRange (peer, start, end, seek, nodes) {
+  _requestRange (peer, start, end, seek, nodes, filter = tautology) {
     const remote = peer.state.bitfield
     const local = this.core.bitfield
 
@@ -339,7 +343,7 @@ class RequestPool {
     if (end === -1) end = peer.state.length
 
     for (let i = start; i < end; i++) {
-      if (!remote.get(i) || local.get(i)) continue
+      if (!filter(i) || !remote.get(i) || local.get(i)) continue
       // TODO: if this was a NO_VALUE request, retry if no blocks can be found elsewhere
       if (this.requests.has(i)) continue
 
@@ -677,8 +681,18 @@ module.exports = class Replicator {
     return promise
   }
 
-  static createRange (start, end, linear) {
-    return new Range(start, end, linear)
+  static createRange (start, end, filter, linear) {
+    // createRange(start, end)
+    if (filter === undefined) {
+      filter = tautology
+
+    // createRange(start, end, linear)
+    } else if (typeof filter === 'boolean') {
+      linear = filter
+      filter = tautology
+    }
+
+    return new Range(start, end, filter, linear)
   }
 
   addRange (range) {
@@ -826,6 +840,10 @@ function pages (core) {
 
 function noop () {}
 
+function tautology () {
+  return true
+}
+
 function log2 (n) {
   let res = 1
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.0.0-alpha.5",
+  "version": "10.0.0-alpha.6",
   "description": "Hypercore 10",
   "main": "index.js",
   "scripts": {

package/test/basic.js

@@ -76,3 +76,15 @@ test('storage options', async function (t) {
   t.alike(await core.get(0), Buffer.from('hello'))
   t.end()
 })
+
+test(
+  'allow publicKeys with different byteLength that 32, if opts.crypto were passed',
+  function (t) {
+    const key = Buffer.alloc(33).fill('a')
+
+    const core = new Hypercore(ram, key, { crypto: {} })
+
+    t.is(core.key, key)
+    t.pass('creating a core with more than 32 byteLength key did not throw')
+  }
+)

package/test/replicate.js

@@ -331,3 +331,42 @@ test('destroying a stream and re-replicating works', async function (t) {
 
   t.is(blocks.length, 33, 'downloaded 33 blocks')
 })
+
+test('replicate discrete range', async function (t) {
+  const a = await create()
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  const b = await create(a.key)
+
+  let d = 0
+  b.on('download', () => d++)
+
+  replicate(a, b, t)
+
+  const r = b.download({ blocks: [0, 2, 3] })
+  await r.downloaded()
+
+  t.is(d, 3)
+  t.alike(await b.get(0), Buffer.from('a'))
+  t.alike(await b.get(2), Buffer.from('c'))
+  t.alike(await b.get(3), Buffer.from('d'))
+})
+
+test('replicate discrete empty range', async function (t) {
+  const a = await create()
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  const b = await create(a.key)
+
+  let d = 0
+  b.on('download', () => d++)
+
+  replicate(a, b, t)
+
+  const r = b.download({ blocks: [] })
+  await r.downloaded()
+
+  t.is(d, 0)
+})