hypercore 10.0.0-alpha.4 → 10.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/index.js +114 -27
- package/lib/block-encryption.js +67 -0
- package/lib/core.js +3 -1
- package/lib/merkle-tree.js +20 -13
- package/lib/replicator.js +45 -13
- package/package.json +4 -3
- package/test/basic.js +12 -0
- package/test/encryption.js +85 -0
- package/test/replicate.js +39 -0
package/README.md
CHANGED
|
@@ -63,7 +63,8 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
|
|
|
63
63
|
createIfMissing: true, // create a new Hypercore key pair if none was present in storage
|
|
64
64
|
overwrite: false, // overwrite any old Hypercore that might already exist
|
|
65
65
|
valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
|
|
66
|
-
keyPair: kp // optionally pass the public key and secret key as a key pair
|
|
66
|
+
keyPair: kp, // optionally pass the public key and secret key as a key pair
|
|
67
|
+
encryptionKey: k // optionally pass an encryption key to enable block encryption
|
|
67
68
|
}
|
|
68
69
|
```
|
|
69
70
|
|
|
@@ -113,6 +114,7 @@ A range can have the following properties:
|
|
|
113
114
|
{
|
|
114
115
|
start: startIndex,
|
|
115
116
|
end: nonInclusiveEndIndex,
|
|
117
|
+
blocks: [index1, index2, ...],
|
|
116
118
|
linear: false // download range linearly and not randomly
|
|
117
119
|
}
|
|
118
120
|
```
|
|
@@ -125,6 +127,12 @@ To download the full core continously (often referred to as non sparse mode) do
|
|
|
125
127
|
core.download({ start: 0, end: -1 })
|
|
126
128
|
```
|
|
127
129
|
|
|
130
|
+
To downloaded a discrete range of blocks pass a list of indices.
|
|
131
|
+
|
|
132
|
+
```js
|
|
133
|
+
core.download({ blocks: [4, 9, 7] });
|
|
134
|
+
```
|
|
135
|
+
|
|
128
136
|
To cancel downloading a range simply destroy the range instance.
|
|
129
137
|
|
|
130
138
|
``` js
|
|
@@ -195,6 +203,10 @@ In contrast to `core.key` this key does not allow you to verify the data but can
|
|
|
195
203
|
|
|
196
204
|
Populated after `ready` has been emitted. Will be `null` before the event.
|
|
197
205
|
|
|
206
|
+
#### `core.encryptionKey`
|
|
207
|
+
|
|
208
|
+
Buffer containing the optional block encryption key of this core. Will be `null` unless block encryption is enabled.
|
|
209
|
+
|
|
198
210
|
#### `core.length`
|
|
199
211
|
|
|
200
212
|
How many blocks of data are available on this core?
|
|
@@ -213,6 +225,10 @@ What is the current fork id of this core?
|
|
|
213
225
|
|
|
214
226
|
Populated after `ready` has been emitted. Will be `0` before the event.
|
|
215
227
|
|
|
228
|
+
#### `core.padding`
|
|
229
|
+
|
|
230
|
+
How much padding is applied to each block of this core? Will be `0` unless block encryption is enabled.
|
|
231
|
+
|
|
216
232
|
#### `const stream = core.replicate(isInitiatorOrReplicationStream)`
|
|
217
233
|
|
|
218
234
|
Create a replication stream. You should pipe this to another Hypercore instance.
|
package/index.js
CHANGED
|
@@ -12,6 +12,7 @@ const fsctl = requireMaybe('fsctl') || { lock: noop, sparse: noop }
|
|
|
12
12
|
const Replicator = require('./lib/replicator')
|
|
13
13
|
const Extensions = require('./lib/extensions')
|
|
14
14
|
const Core = require('./lib/core')
|
|
15
|
+
const BlockEncryption = require('./lib/block-encryption')
|
|
15
16
|
|
|
16
17
|
const promises = Symbol.for('hypercore.promises')
|
|
17
18
|
const inspect = Symbol.for('nodejs.util.inspect.custom')
|
|
@@ -28,14 +29,17 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
28
29
|
opts = key
|
|
29
30
|
key = null
|
|
30
31
|
}
|
|
32
|
+
|
|
31
33
|
if (key && typeof key === 'string') {
|
|
32
34
|
key = Buffer.from(key, 'hex')
|
|
33
35
|
}
|
|
34
|
-
|
|
36
|
+
|
|
37
|
+
if (!opts) opts = {}
|
|
38
|
+
|
|
39
|
+
if (!opts.crypto && key && key.byteLength !== 32) {
|
|
35
40
|
throw new Error('Hypercore key should be 32 bytes')
|
|
36
41
|
}
|
|
37
42
|
|
|
38
|
-
if (!opts) opts = {}
|
|
39
43
|
if (!storage) storage = opts.storage
|
|
40
44
|
|
|
41
45
|
this[promises] = true
|
|
@@ -44,6 +48,7 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
44
48
|
this.crypto = opts.crypto || hypercoreCrypto
|
|
45
49
|
this.core = null
|
|
46
50
|
this.replicator = null
|
|
51
|
+
this.encryption = null
|
|
47
52
|
this.extensions = opts.extensions || new Extensions()
|
|
48
53
|
this.cache = opts.cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (opts.cache || null)
|
|
49
54
|
|
|
@@ -61,6 +66,8 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
61
66
|
this.closing = null
|
|
62
67
|
this.opening = opts._opening || this._open(key, storage, opts)
|
|
63
68
|
this.opening.catch(noop)
|
|
69
|
+
|
|
70
|
+
this._preappend = preappend.bind(this)
|
|
64
71
|
}
|
|
65
72
|
|
|
66
73
|
[inspect] (depth, opts) {
|
|
@@ -135,6 +142,7 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
135
142
|
this.discoveryKey = o.discoveryKey
|
|
136
143
|
this.core = o.core
|
|
137
144
|
this.replicator = o.replicator
|
|
145
|
+
this.encryption = o.encryption
|
|
138
146
|
this.writable = !!this.sign
|
|
139
147
|
this.autoClose = o.autoClose
|
|
140
148
|
}
|
|
@@ -204,7 +212,7 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
204
212
|
}
|
|
205
213
|
|
|
206
214
|
get byteLength () {
|
|
207
|
-
return this.core === null ? 0 : this.core.tree.byteLength
|
|
215
|
+
return this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding)
|
|
208
216
|
}
|
|
209
217
|
|
|
210
218
|
get fork () {
|
|
@@ -215,6 +223,14 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
215
223
|
return this.replicator === null ? [] : this.replicator.peers
|
|
216
224
|
}
|
|
217
225
|
|
|
226
|
+
get encryptionKey () {
|
|
227
|
+
return this.encryption && this.encryption.key
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
get padding () {
|
|
231
|
+
return this.encryption === null ? 0 : this.encryption.padding
|
|
232
|
+
}
|
|
233
|
+
|
|
218
234
|
ready () {
|
|
219
235
|
return this.opening
|
|
220
236
|
}
|
|
@@ -268,6 +284,10 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
268
284
|
this.key = this.core.header.signer.publicKey
|
|
269
285
|
this.writable = !!this.sign
|
|
270
286
|
|
|
287
|
+
if (!this.encryption && opts.encryptionKey) {
|
|
288
|
+
this.encryption = new BlockEncryption(opts.encryptionKey, this.key)
|
|
289
|
+
}
|
|
290
|
+
|
|
271
291
|
this.extensions.attach(this.replicator)
|
|
272
292
|
this.opened = true
|
|
273
293
|
|
|
@@ -302,6 +322,12 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
302
322
|
}
|
|
303
323
|
|
|
304
324
|
if (value) {
|
|
325
|
+
if (this.encryption) {
|
|
326
|
+
this.encryption.decrypt(bitfield.start, value)
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
value = value.subarray(this.padding)
|
|
330
|
+
|
|
305
331
|
for (let i = 0; i < this.sessions.length; i++) {
|
|
306
332
|
this.sessions[i].emit('download', bitfield.start, value, from)
|
|
307
333
|
}
|
|
@@ -340,7 +366,7 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
340
366
|
async seek (bytes) {
|
|
341
367
|
if (this.opened === false) await this.opening
|
|
342
368
|
|
|
343
|
-
const s = this.core.tree.seek(bytes)
|
|
369
|
+
const s = this.core.tree.seek(bytes, this.padding)
|
|
344
370
|
|
|
345
371
|
return (await s.update()) || this.replicator.requestSeek(s)
|
|
346
372
|
}
|
|
@@ -364,20 +390,41 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
364
390
|
async _get (index, opts) {
|
|
365
391
|
const encoding = (opts && opts.valueEncoding && c.from(codecs(opts.valueEncoding))) || this.valueEncoding
|
|
366
392
|
|
|
367
|
-
|
|
368
|
-
|
|
393
|
+
let block
|
|
394
|
+
|
|
395
|
+
if (this.core.bitfield.get(index)) {
|
|
396
|
+
block = await this.core.blocks.get(index)
|
|
397
|
+
} else {
|
|
398
|
+
if (opts && opts.onwait) opts.onwait(index)
|
|
399
|
+
block = await this.replicator.requestBlock(index)
|
|
400
|
+
}
|
|
369
401
|
|
|
370
|
-
|
|
402
|
+
if (this.encryption) this.encryption.decrypt(index, block)
|
|
403
|
+
return this._decode(encoding, block)
|
|
371
404
|
}
|
|
372
405
|
|
|
373
406
|
download (range) {
|
|
374
|
-
const start = (range && range.start) || 0
|
|
375
|
-
const end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
|
|
376
407
|
const linear = !!(range && range.linear)
|
|
377
408
|
|
|
378
|
-
|
|
409
|
+
let start
|
|
410
|
+
let end
|
|
411
|
+
let filter
|
|
412
|
+
|
|
413
|
+
if (range && range.blocks) {
|
|
414
|
+
const blocks = range.blocks instanceof Set
|
|
415
|
+
? range.blocks
|
|
416
|
+
: new Set(range.blocks)
|
|
417
|
+
|
|
418
|
+
start = range.start || (blocks.size ? min(range.blocks) : 0)
|
|
419
|
+
end = range.end || (blocks.size ? max(range.blocks) + 1 : 0)
|
|
420
|
+
|
|
421
|
+
filter = (i) => blocks.has(i)
|
|
422
|
+
} else {
|
|
423
|
+
start = (range && range.start) || 0
|
|
424
|
+
end = typeof (range && range.end) === 'number' ? range.end : -1 // download all
|
|
425
|
+
}
|
|
379
426
|
|
|
380
|
-
const r = Replicator.createRange(start, end, linear)
|
|
427
|
+
const r = Replicator.createRange(start, end, filter, linear)
|
|
381
428
|
|
|
382
429
|
if (this.opened) this.replicator.addRange(r)
|
|
383
430
|
else this.opening.then(() => this.replicator.addRange(r), noop)
|
|
@@ -410,22 +457,16 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
410
457
|
if (this.opened === false) await this.opening
|
|
411
458
|
if (this.writable === false) throw new Error('Core is not writable')
|
|
412
459
|
|
|
413
|
-
|
|
414
|
-
const buffers = new Array(blks.length)
|
|
460
|
+
blocks = Array.isArray(blocks) ? blocks : [blocks]
|
|
415
461
|
|
|
416
|
-
|
|
417
|
-
|
|
462
|
+
const preappend = this.encryption && this._preappend
|
|
463
|
+
const buffers = new Array(blocks.length)
|
|
418
464
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
: this.valueEncoding
|
|
422
|
-
? c.encode(this.valueEncoding, blk)
|
|
423
|
-
: Buffer.from(blk)
|
|
424
|
-
|
|
425
|
-
buffers[i] = buf
|
|
465
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
466
|
+
buffers[i] = this._encode(this.valueEncoding, blocks[i])
|
|
426
467
|
}
|
|
427
468
|
|
|
428
|
-
return await this.core.append(buffers, this.sign)
|
|
469
|
+
return await this.core.append(buffers, this.sign, { preappend })
|
|
429
470
|
}
|
|
430
471
|
|
|
431
472
|
registerExtension (name, handlers) {
|
|
@@ -436,14 +477,38 @@ module.exports = class Hypercore extends EventEmitter {
|
|
|
436
477
|
onextensionupdate () {
|
|
437
478
|
if (this.replicator !== null) this.replicator.broadcastOptions()
|
|
438
479
|
}
|
|
439
|
-
}
|
|
440
480
|
|
|
441
|
-
|
|
481
|
+
_encode (enc, val) {
|
|
482
|
+
const state = { start: this.padding, end: this.padding, buffer: null }
|
|
483
|
+
|
|
484
|
+
if (Buffer.isBuffer(val)) {
|
|
485
|
+
if (state.start === 0) return val
|
|
486
|
+
state.end += val.byteLength
|
|
487
|
+
} else if (enc) {
|
|
488
|
+
enc.preencode(state, val)
|
|
489
|
+
} else {
|
|
490
|
+
val = Buffer.from(val)
|
|
491
|
+
if (state.start === 0) return val
|
|
492
|
+
state.end += val.byteLength
|
|
493
|
+
}
|
|
442
494
|
|
|
443
|
-
|
|
444
|
-
|
|
495
|
+
state.buffer = Buffer.allocUnsafe(state.end)
|
|
496
|
+
|
|
497
|
+
if (enc) enc.encode(state, val)
|
|
498
|
+
else state.buffer.set(val, state.start)
|
|
499
|
+
|
|
500
|
+
return state.buffer
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
_decode (enc, block) {
|
|
504
|
+
block = block.subarray(this.padding)
|
|
505
|
+
if (enc) return c.decode(enc, block)
|
|
506
|
+
return block
|
|
507
|
+
}
|
|
445
508
|
}
|
|
446
509
|
|
|
510
|
+
function noop () {}
|
|
511
|
+
|
|
447
512
|
function isStream (s) {
|
|
448
513
|
return typeof s === 'object' && s && typeof s.pipe === 'function'
|
|
449
514
|
}
|
|
@@ -459,3 +524,25 @@ function requireMaybe (name) {
|
|
|
459
524
|
function toHex (buf) {
|
|
460
525
|
return buf && buf.toString('hex')
|
|
461
526
|
}
|
|
527
|
+
|
|
528
|
+
function reduce (iter, fn, acc) {
|
|
529
|
+
for (const item of iter) acc = fn(acc, item)
|
|
530
|
+
return acc
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function min (arr) {
|
|
534
|
+
return reduce(arr, (a, b) => Math.min(a, b), Infinity)
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
function max (arr) {
|
|
538
|
+
return reduce(arr, (a, b) => Math.max(a, b), -Infinity)
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function preappend (blocks) {
|
|
542
|
+
const offset = this.core.tree.length
|
|
543
|
+
const fork = this.core.tree.fork
|
|
544
|
+
|
|
545
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
546
|
+
this.encryption.encrypt(offset + i, blocks[i], fork)
|
|
547
|
+
}
|
|
548
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
const sodium = require('sodium-universal')
|
|
2
|
+
const c = require('compact-encoding')
|
|
3
|
+
|
|
4
|
+
const nonce = Buffer.alloc(sodium.crypto_stream_NONCEBYTES)
|
|
5
|
+
|
|
6
|
+
module.exports = class BlockEncryption {
|
|
7
|
+
constructor (encryptionKey, hypercoreKey) {
|
|
8
|
+
const subKeys = Buffer.alloc(2 * sodium.crypto_stream_KEYBYTES)
|
|
9
|
+
|
|
10
|
+
this.key = encryptionKey
|
|
11
|
+
this.blockKey = subKeys.subarray(0, sodium.crypto_stream_KEYBYTES)
|
|
12
|
+
this.blindingKey = subKeys.subarray(sodium.crypto_stream_KEYBYTES)
|
|
13
|
+
this.padding = 8
|
|
14
|
+
|
|
15
|
+
sodium.crypto_generichash(this.blockKey, encryptionKey, hypercoreKey)
|
|
16
|
+
sodium.crypto_generichash(this.blindingKey, this.blockKey)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
encrypt (index, block, fork) {
|
|
20
|
+
const padding = block.subarray(0, this.padding)
|
|
21
|
+
block = block.subarray(this.padding)
|
|
22
|
+
|
|
23
|
+
c.uint64.encode({ start: 0, end: 8, buffer: padding }, fork)
|
|
24
|
+
c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
|
|
25
|
+
|
|
26
|
+
// Zero out any previous padding.
|
|
27
|
+
nonce.fill(0, 8, 8 + padding.byteLength)
|
|
28
|
+
|
|
29
|
+
// Blind the fork ID, possibly risking reusing the nonce on a reorg of the
|
|
30
|
+
// Hypercore. This is fine as the blinding is best-effort and the latest
|
|
31
|
+
// fork ID shared on replication anyway.
|
|
32
|
+
sodium.crypto_stream_xor(
|
|
33
|
+
padding,
|
|
34
|
+
padding,
|
|
35
|
+
nonce,
|
|
36
|
+
this.blindingKey
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
nonce.set(padding, 8)
|
|
40
|
+
|
|
41
|
+
// The combination of a (blinded) fork ID and a block index is unique for a
|
|
42
|
+
// given Hypercore and is therefore a valid nonce for encrypting the block.
|
|
43
|
+
sodium.crypto_stream_xor(
|
|
44
|
+
block,
|
|
45
|
+
block,
|
|
46
|
+
nonce,
|
|
47
|
+
this.blockKey
|
|
48
|
+
)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
decrypt (index, block) {
|
|
52
|
+
const padding = block.subarray(0, this.padding)
|
|
53
|
+
block = block.subarray(this.padding)
|
|
54
|
+
|
|
55
|
+
c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
|
|
56
|
+
|
|
57
|
+
nonce.set(padding, 8)
|
|
58
|
+
|
|
59
|
+
// Decrypt the block using the blinded fork ID.
|
|
60
|
+
sodium.crypto_stream_xor(
|
|
61
|
+
block,
|
|
62
|
+
block,
|
|
63
|
+
nonce,
|
|
64
|
+
this.blockKey
|
|
65
|
+
)
|
|
66
|
+
}
|
|
67
|
+
}
|
package/lib/core.js
CHANGED
|
@@ -214,10 +214,12 @@ module.exports = class Core {
|
|
|
214
214
|
}
|
|
215
215
|
}
|
|
216
216
|
|
|
217
|
-
async append (values, sign = this.defaultSign) {
|
|
217
|
+
async append (values, sign = this.defaultSign, hooks = {}) {
|
|
218
218
|
await this._mutex.lock()
|
|
219
219
|
|
|
220
220
|
try {
|
|
221
|
+
if (hooks.preappend) await hooks.preappend(values)
|
|
222
|
+
|
|
221
223
|
if (!values.length) return this.tree.length
|
|
222
224
|
|
|
223
225
|
const batch = this.tree.batch()
|
package/lib/merkle-tree.js
CHANGED
|
@@ -272,11 +272,15 @@ class ReorgBatch extends MerkleTreeBatch {
|
|
|
272
272
|
}
|
|
273
273
|
|
|
274
274
|
class ByteSeeker {
|
|
275
|
-
constructor (tree, bytes) {
|
|
275
|
+
constructor (tree, bytes, padding = 0) {
|
|
276
276
|
this.tree = tree
|
|
277
277
|
this.bytes = bytes
|
|
278
|
-
this.
|
|
279
|
-
|
|
278
|
+
this.padding = padding
|
|
279
|
+
|
|
280
|
+
const size = tree.byteLength - (tree.length * padding)
|
|
281
|
+
|
|
282
|
+
this.start = bytes >= size ? tree.length : 0
|
|
283
|
+
this.end = bytes < size ? tree.length : 0
|
|
280
284
|
}
|
|
281
285
|
|
|
282
286
|
nodes () {
|
|
@@ -287,12 +291,12 @@ class ByteSeeker {
|
|
|
287
291
|
if (!bytes) return [0, 0]
|
|
288
292
|
|
|
289
293
|
for (const node of this.tree.roots) { // all async ticks happen once we find the root so safe
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
}
|
|
294
|
+
let size = node.size
|
|
295
|
+
if (this.padding > 0) size -= this.padding * flat.countLeaves(node.index)
|
|
293
296
|
|
|
294
|
-
if (bytes
|
|
295
|
-
|
|
297
|
+
if (bytes === size) return [flat.rightSpan(node.index) + 2, 0]
|
|
298
|
+
if (bytes > size) {
|
|
299
|
+
bytes -= size
|
|
296
300
|
continue
|
|
297
301
|
}
|
|
298
302
|
|
|
@@ -301,9 +305,12 @@ class ByteSeeker {
|
|
|
301
305
|
while ((ite.index & 1) !== 0) {
|
|
302
306
|
const l = await this.tree.get(ite.leftChild(), false)
|
|
303
307
|
if (l) {
|
|
304
|
-
|
|
305
|
-
if (
|
|
306
|
-
|
|
308
|
+
let size = l.size
|
|
309
|
+
if (this.padding > 0) size -= this.padding * ite.countLeaves()
|
|
310
|
+
|
|
311
|
+
if (size === bytes) return [ite.rightSpan() + 2, 0]
|
|
312
|
+
if (size > bytes) continue
|
|
313
|
+
bytes -= size
|
|
307
314
|
ite.sibling()
|
|
308
315
|
} else {
|
|
309
316
|
ite.parent()
|
|
@@ -355,8 +362,8 @@ module.exports = class MerkleTree {
|
|
|
355
362
|
return new MerkleTreeBatch(this)
|
|
356
363
|
}
|
|
357
364
|
|
|
358
|
-
seek (bytes) {
|
|
359
|
-
return new ByteSeeker(this, bytes)
|
|
365
|
+
seek (bytes, padding) {
|
|
366
|
+
return new ByteSeeker(this, bytes, padding)
|
|
360
367
|
}
|
|
361
368
|
|
|
362
369
|
hash () {
|
package/lib/replicator.js
CHANGED
|
@@ -25,11 +25,12 @@ class InvertedPromise {
|
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
class Request {
|
|
28
|
-
constructor (index, seek) {
|
|
28
|
+
constructor (index, seek, nodes) {
|
|
29
29
|
this.peer = null
|
|
30
30
|
this.index = index
|
|
31
31
|
this.seek = seek
|
|
32
32
|
this.value = seek === 0
|
|
33
|
+
this.nodes = nodes
|
|
33
34
|
this.promises = []
|
|
34
35
|
}
|
|
35
36
|
|
|
@@ -119,9 +120,10 @@ class Seek {
|
|
|
119
120
|
}
|
|
120
121
|
|
|
121
122
|
class Range {
|
|
122
|
-
constructor (start, end, linear) {
|
|
123
|
+
constructor (start, end, filter, linear) {
|
|
123
124
|
this.start = start
|
|
124
125
|
this.end = end
|
|
126
|
+
this.filter = filter
|
|
125
127
|
this.linear = !!linear
|
|
126
128
|
this.promise = null
|
|
127
129
|
this.done = false
|
|
@@ -144,7 +146,7 @@ class Range {
|
|
|
144
146
|
}
|
|
145
147
|
|
|
146
148
|
for (; this._start < this.end; this._start++) {
|
|
147
|
-
if (!bitfield.get(this._start)) return false
|
|
149
|
+
if (this.filter(this._start) && !bitfield.get(this._start)) return false
|
|
148
150
|
}
|
|
149
151
|
|
|
150
152
|
return true
|
|
@@ -278,7 +280,9 @@ class RequestPool {
|
|
|
278
280
|
|
|
279
281
|
_updateSeek (peer, seek) {
|
|
280
282
|
if (seek.request) return false
|
|
281
|
-
|
|
283
|
+
// We have to snapshot the nodes here now, due to the caching of the request...
|
|
284
|
+
const nodes = log2(seek.seeker.end - seek.seeker.start)
|
|
285
|
+
seek.request = this._requestRange(peer, seek.seeker.start, seek.seeker.end, seek.seeker.bytes, nodes)
|
|
282
286
|
return seek.request !== null
|
|
283
287
|
}
|
|
284
288
|
|
|
@@ -296,10 +300,13 @@ class RequestPool {
|
|
|
296
300
|
const end = range.end === -1 ? peer.state.length : range.end
|
|
297
301
|
if (end <= range._start) return false
|
|
298
302
|
|
|
299
|
-
if (range.linear) return !!this._requestRange(peer, range._start, end, 0)
|
|
303
|
+
if (range.linear) return !!this._requestRange(peer, range._start, end, 0, 0, range.filter)
|
|
300
304
|
|
|
301
305
|
const r = range._start + Math.floor(Math.random() * (end - range._start))
|
|
302
|
-
return !!(
|
|
306
|
+
return !!(
|
|
307
|
+
this._requestRange(peer, r, end, 0, 0, range.filter) ||
|
|
308
|
+
this._requestRange(peer, range._start, r, 0, 0, range.filter)
|
|
309
|
+
)
|
|
303
310
|
}
|
|
304
311
|
|
|
305
312
|
_updateUpgrade (peer) {
|
|
@@ -328,7 +335,7 @@ class RequestPool {
|
|
|
328
335
|
this.pendingUpgrade = null
|
|
329
336
|
}
|
|
330
337
|
|
|
331
|
-
_requestRange (peer, start, end, seek) {
|
|
338
|
+
_requestRange (peer, start, end, seek, nodes, filter = tautology) {
|
|
332
339
|
const remote = peer.state.bitfield
|
|
333
340
|
const local = this.core.bitfield
|
|
334
341
|
|
|
@@ -336,12 +343,12 @@ class RequestPool {
|
|
|
336
343
|
if (end === -1) end = peer.state.length
|
|
337
344
|
|
|
338
345
|
for (let i = start; i < end; i++) {
|
|
339
|
-
if (!remote.get(i) || local.get(i)) continue
|
|
346
|
+
if (!filter(i) || !remote.get(i) || local.get(i)) continue
|
|
340
347
|
// TODO: if this was a NO_VALUE request, retry if no blocks can be found elsewhere
|
|
341
348
|
if (this.requests.has(i)) continue
|
|
342
349
|
|
|
343
350
|
// TODO: if seeking and i >= core.length, let that takes precendance in the upgrade req
|
|
344
|
-
const req = new Request(i, i < this.core.tree.length ? seek : 0)
|
|
351
|
+
const req = new Request(i, i < this.core.tree.length ? seek : 0, nodes)
|
|
345
352
|
this.requests.set(i, req)
|
|
346
353
|
this.send(peer, req)
|
|
347
354
|
return req
|
|
@@ -390,7 +397,7 @@ class RequestPool {
|
|
|
390
397
|
|
|
391
398
|
if (data.block.index < this.core.tree.length || this.core.truncating > 0) {
|
|
392
399
|
try {
|
|
393
|
-
data.block.nodes = await this.core.tree.nodes(data.block.index * 2)
|
|
400
|
+
data.block.nodes = Math.max(req.nodes, await this.core.tree.nodes(data.block.index * 2))
|
|
394
401
|
} catch (err) {
|
|
395
402
|
console.error('TODO handle me:', err.stack)
|
|
396
403
|
}
|
|
@@ -606,7 +613,7 @@ class RequestPool {
|
|
|
606
613
|
return e.createPromise()
|
|
607
614
|
}
|
|
608
615
|
|
|
609
|
-
const r = new Request(index, 0)
|
|
616
|
+
const r = new Request(index, 0, 0)
|
|
610
617
|
|
|
611
618
|
this.requests.set(index, r)
|
|
612
619
|
this.pending.push(r)
|
|
@@ -674,8 +681,18 @@ module.exports = class Replicator {
|
|
|
674
681
|
return promise
|
|
675
682
|
}
|
|
676
683
|
|
|
677
|
-
static createRange (start, end, linear) {
|
|
678
|
-
|
|
684
|
+
static createRange (start, end, filter, linear) {
|
|
685
|
+
// createRange(start, end)
|
|
686
|
+
if (filter === undefined) {
|
|
687
|
+
filter = tautology
|
|
688
|
+
|
|
689
|
+
// createRange(start, end, linear)
|
|
690
|
+
} else if (typeof filter === 'boolean') {
|
|
691
|
+
linear = filter
|
|
692
|
+
filter = tautology
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
return new Range(start, end, filter, linear)
|
|
679
696
|
}
|
|
680
697
|
|
|
681
698
|
addRange (range) {
|
|
@@ -822,3 +839,18 @@ function pages (core) {
|
|
|
822
839
|
}
|
|
823
840
|
|
|
824
841
|
function noop () {}
|
|
842
|
+
|
|
843
|
+
function tautology () {
|
|
844
|
+
return true
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
function log2 (n) {
|
|
848
|
+
let res = 1
|
|
849
|
+
|
|
850
|
+
while (n > 2) {
|
|
851
|
+
n /= 2
|
|
852
|
+
res++
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
return res
|
|
856
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hypercore",
|
|
3
|
-
"version": "10.0.0-alpha.
|
|
3
|
+
"version": "10.0.0-alpha.8",
|
|
4
4
|
"description": "Hypercore 10",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -31,14 +31,15 @@
|
|
|
31
31
|
"@hyperswarm/secret-stream": "^5.0.0",
|
|
32
32
|
"big-sparse-array": "^1.0.2",
|
|
33
33
|
"codecs": "^2.2.0",
|
|
34
|
-
"compact-encoding": "^2.
|
|
34
|
+
"compact-encoding": "^2.5.0",
|
|
35
35
|
"crc32-universal": "^1.0.1",
|
|
36
|
-
"flat-tree": "^1.
|
|
36
|
+
"flat-tree": "^1.9.0",
|
|
37
37
|
"hypercore-crypto": "^2.1.1",
|
|
38
38
|
"is-options": "^1.0.1",
|
|
39
39
|
"random-access-file": "^2.1.4",
|
|
40
40
|
"random-array-iterator": "^1.0.0",
|
|
41
41
|
"safety-catch": "^1.0.1",
|
|
42
|
+
"sodium-universal": "^3.0.4",
|
|
42
43
|
"uint64le": "^1.0.0",
|
|
43
44
|
"xache": "^1.0.0"
|
|
44
45
|
},
|
package/test/basic.js
CHANGED
|
@@ -76,3 +76,15 @@ test('storage options', async function (t) {
|
|
|
76
76
|
t.alike(await core.get(0), Buffer.from('hello'))
|
|
77
77
|
t.end()
|
|
78
78
|
})
|
|
79
|
+
|
|
80
|
+
test(
|
|
81
|
+
'allow publicKeys with different byteLength that 32, if opts.crypto were passed',
|
|
82
|
+
function (t) {
|
|
83
|
+
const key = Buffer.alloc(33).fill('a')
|
|
84
|
+
|
|
85
|
+
const core = new Hypercore(ram, key, { crypto: {} })
|
|
86
|
+
|
|
87
|
+
t.is(core.key, key)
|
|
88
|
+
t.pass('creating a core with more than 32 byteLength key did not throw')
|
|
89
|
+
}
|
|
90
|
+
)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
const test = require('brittle')
|
|
2
|
+
const { create, replicate } = require('./helpers')
|
|
3
|
+
|
|
4
|
+
const encryptionKey = Buffer.alloc(32, 'hello world')
|
|
5
|
+
|
|
6
|
+
test('encrypted append and get', async function (t) {
|
|
7
|
+
const a = await create({ encryptionKey })
|
|
8
|
+
|
|
9
|
+
t.alike(a.encryptionKey, encryptionKey)
|
|
10
|
+
|
|
11
|
+
await a.append(['hello'])
|
|
12
|
+
|
|
13
|
+
t.is(a.byteLength, 5)
|
|
14
|
+
t.is(a.core.tree.byteLength, 5 + a.padding)
|
|
15
|
+
|
|
16
|
+
const unencrypted = await a.get(0)
|
|
17
|
+
const encrypted = await a.core.blocks.get(0)
|
|
18
|
+
|
|
19
|
+
t.alike(unencrypted, Buffer.from('hello'))
|
|
20
|
+
t.unlike(unencrypted, encrypted)
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
test('encrypted seek', async function (t) {
|
|
24
|
+
const a = await create({ encryptionKey })
|
|
25
|
+
|
|
26
|
+
await a.append(['hello', 'world', '!'])
|
|
27
|
+
|
|
28
|
+
t.alike(await a.seek(0), [0, 0])
|
|
29
|
+
t.alike(await a.seek(4), [0, 4])
|
|
30
|
+
t.alike(await a.seek(5), [1, 0])
|
|
31
|
+
t.alike(await a.seek(6), [1, 1])
|
|
32
|
+
t.alike(await a.seek(6), [1, 1])
|
|
33
|
+
t.alike(await a.seek(9), [1, 4])
|
|
34
|
+
t.alike(await a.seek(10), [2, 0])
|
|
35
|
+
t.alike(await a.seek(11), [3, 0])
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
test('encrypted replication', async function (t) {
|
|
39
|
+
const a = await create({ encryptionKey })
|
|
40
|
+
|
|
41
|
+
await a.append(['a', 'b', 'c', 'd', 'e'])
|
|
42
|
+
|
|
43
|
+
t.test('with encryption key', async function (t) {
|
|
44
|
+
t.plan(10)
|
|
45
|
+
|
|
46
|
+
const b = await create(a.key, { encryptionKey })
|
|
47
|
+
|
|
48
|
+
b.on('download', (i, block) => {
|
|
49
|
+
t.alike(block, Buffer.from([i + /* a */ 0x61]))
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
replicate(a, b, t)
|
|
53
|
+
|
|
54
|
+
const r = b.download({ start: 0, end: a.length })
|
|
55
|
+
await r.downloaded()
|
|
56
|
+
|
|
57
|
+
for (let i = 0; i < 5; i++) {
|
|
58
|
+
t.alike(await b.get(i), await a.get(i))
|
|
59
|
+
}
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
t.test('without encryption key', async function (t) {
|
|
63
|
+
const b = await create(a.key)
|
|
64
|
+
|
|
65
|
+
replicate(a, b, t)
|
|
66
|
+
|
|
67
|
+
const r = b.download({ start: 0, end: a.length })
|
|
68
|
+
await r.downloaded()
|
|
69
|
+
|
|
70
|
+
for (let i = 0; i < 5; i++) {
|
|
71
|
+
t.unlike(await b.get(i), await a.get(i))
|
|
72
|
+
t.alike(await b.get(i), await a.core.blocks.get(i))
|
|
73
|
+
}
|
|
74
|
+
})
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
test('encrypted sessions', async function (t) {
|
|
78
|
+
const a = await create({ encryptionKey })
|
|
79
|
+
|
|
80
|
+
await a.append(['hello'])
|
|
81
|
+
|
|
82
|
+
const session = a.session()
|
|
83
|
+
|
|
84
|
+
t.alike(await session.get(0), Buffer.from('hello'))
|
|
85
|
+
})
|
package/test/replicate.js
CHANGED
|
@@ -331,3 +331,42 @@ test('destroying a stream and re-replicating works', async function (t) {
|
|
|
331
331
|
|
|
332
332
|
t.is(blocks.length, 33, 'downloaded 33 blocks')
|
|
333
333
|
})
|
|
334
|
+
|
|
335
|
+
test('replicate discrete range', async function (t) {
|
|
336
|
+
const a = await create()
|
|
337
|
+
|
|
338
|
+
await a.append(['a', 'b', 'c', 'd', 'e'])
|
|
339
|
+
|
|
340
|
+
const b = await create(a.key)
|
|
341
|
+
|
|
342
|
+
let d = 0
|
|
343
|
+
b.on('download', () => d++)
|
|
344
|
+
|
|
345
|
+
replicate(a, b, t)
|
|
346
|
+
|
|
347
|
+
const r = b.download({ blocks: [0, 2, 3] })
|
|
348
|
+
await r.downloaded()
|
|
349
|
+
|
|
350
|
+
t.is(d, 3)
|
|
351
|
+
t.alike(await b.get(0), Buffer.from('a'))
|
|
352
|
+
t.alike(await b.get(2), Buffer.from('c'))
|
|
353
|
+
t.alike(await b.get(3), Buffer.from('d'))
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
test('replicate discrete empty range', async function (t) {
|
|
357
|
+
const a = await create()
|
|
358
|
+
|
|
359
|
+
await a.append(['a', 'b', 'c', 'd', 'e'])
|
|
360
|
+
|
|
361
|
+
const b = await create(a.key)
|
|
362
|
+
|
|
363
|
+
let d = 0
|
|
364
|
+
b.on('download', () => d++)
|
|
365
|
+
|
|
366
|
+
replicate(a, b, t)
|
|
367
|
+
|
|
368
|
+
const r = b.download({ blocks: [] })
|
|
369
|
+
await r.downloaded()
|
|
370
|
+
|
|
371
|
+
t.is(d, 0)
|
|
372
|
+
})
|