hypercore 11.16.2 → 11.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -29
- package/index.js +212 -127
- package/lib/audit.js +17 -6
- package/lib/bit-interlude.js +17 -7
- package/lib/bitfield.js +72 -52
- package/lib/caps.js +5 -1
- package/lib/copy-prologue.js +14 -10
- package/lib/core.js +109 -56
- package/lib/default-encryption.js +14 -28
- package/lib/download.js +10 -10
- package/lib/fully-remote-proof.js +3 -3
- package/lib/hotswap-queue.js +5 -5
- package/lib/info.js +4 -4
- package/lib/merkle-tree.js +143 -104
- package/lib/messages.js +163 -143
- package/lib/multisig.js +19 -12
- package/lib/mutex.js +9 -7
- package/lib/receiver-queue.js +6 -6
- package/lib/remote-bitfield.js +30 -32
- package/lib/replicator.js +383 -265
- package/lib/session-state.js +112 -75
- package/lib/streams.js +16 -16
- package/lib/verifier.js +69 -43
- package/package.json +5 -3
package/README.md
CHANGED
|
@@ -8,11 +8,11 @@ Built for sharing large datasets and streams of real time data
|
|
|
8
8
|
|
|
9
9
|
## Features
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
- **Sparse replication.** Only download the data you are interested in.
|
|
12
|
+
- **Realtime.** Get the latest updates to the log fast and securely.
|
|
13
|
+
- **Performant.** Uses a simple flat file structure to maximize I/O performance.
|
|
14
|
+
- **Secure.** Uses signed merkle trees to verify log integrity in real time.
|
|
15
|
+
- **Modular.** Hypercore aims to do one thing and one thing well - distributing a stream of data.
|
|
16
16
|
|
|
17
17
|
Note that the latest release is Hypercore 10, which adds support for truncate and many other things.
|
|
18
18
|
Version 10 is not compatible with earlier versions (9 and earlier), but is considered LTS, meaning the storage format and wire protocol is forward compatible with future versions.
|
|
@@ -31,7 +31,7 @@ Make a new Hypercore instance.
|
|
|
31
31
|
|
|
32
32
|
`storage` should be set to a directory where you want to store the data and core metadata.
|
|
33
33
|
|
|
34
|
-
```
|
|
34
|
+
```js
|
|
35
35
|
const core = new Hypercore('./directory') // store data in ./directory
|
|
36
36
|
```
|
|
37
37
|
|
|
@@ -41,7 +41,7 @@ Alternatively you can pass a [Hypercore Storage](https://github.com/holepunchto/
|
|
|
41
41
|
|
|
42
42
|
`options` include:
|
|
43
43
|
|
|
44
|
-
```
|
|
44
|
+
```js
|
|
45
45
|
{
|
|
46
46
|
createIfMissing: true, // create a new Hypercore key pair if none was present in storage
|
|
47
47
|
overwrite: false, // overwrite any old Hypercore that might already exist
|
|
@@ -114,7 +114,7 @@ Signers are an array of objects with the following structure:
|
|
|
114
114
|
Append a block of data (or an array of blocks) to the core.
|
|
115
115
|
Returns the new length and byte length of the core.
|
|
116
116
|
|
|
117
|
-
```
|
|
117
|
+
```js
|
|
118
118
|
// simple call append with a new block of data
|
|
119
119
|
await core.append(Buffer.from('I am a block of data'))
|
|
120
120
|
|
|
@@ -138,7 +138,7 @@ await core.append([Buffer.from('batch block 1'), Buffer.from('batch block 2')])
|
|
|
138
138
|
Get a block of data.
|
|
139
139
|
If the data is not available locally this method will prioritize and wait for the data to be downloaded.
|
|
140
140
|
|
|
141
|
-
```
|
|
141
|
+
```js
|
|
142
142
|
// get block #42
|
|
143
143
|
const block = await core.get(42)
|
|
144
144
|
|
|
@@ -151,7 +151,7 @@ const blockLocal = await core.get(44, { wait: false })
|
|
|
151
151
|
|
|
152
152
|
`options` include:
|
|
153
153
|
|
|
154
|
-
```
|
|
154
|
+
```js
|
|
155
155
|
{
|
|
156
156
|
wait: true, // wait for block to be downloaded
|
|
157
157
|
onwait: () => {}, // hook that is called if the get is waiting for download
|
|
@@ -171,7 +171,7 @@ Check if the core has all blocks between `start` and `end`.
|
|
|
171
171
|
|
|
172
172
|
Waits for initial proof of the new core length until all `findingPeers` calls has finished.
|
|
173
173
|
|
|
174
|
-
```
|
|
174
|
+
```js
|
|
175
175
|
const updated = await core.update()
|
|
176
176
|
|
|
177
177
|
console.log('core was updated?', updated, 'length is', core.length)
|
|
@@ -179,7 +179,7 @@ console.log('core was updated?', updated, 'length is', core.length)
|
|
|
179
179
|
|
|
180
180
|
`options` include:
|
|
181
181
|
|
|
182
|
-
```
|
|
182
|
+
```js
|
|
183
183
|
{
|
|
184
184
|
wait: false,
|
|
185
185
|
activeRequests: undefined, // Advanced option. Pass requests for replicating blocks
|
|
@@ -196,7 +196,7 @@ Seek to a byte offset.
|
|
|
196
196
|
Returns `[index, relativeOffset]`, where `index` is the data block the `byteOffset` is contained in and `relativeOffset` is
|
|
197
197
|
the relative byte offset in the data block.
|
|
198
198
|
|
|
199
|
-
```
|
|
199
|
+
```js
|
|
200
200
|
await core.append([Buffer.from('abc'), Buffer.from('d'), Buffer.from('efg')])
|
|
201
201
|
|
|
202
202
|
const first = await core.seek(1) // returns [0, 1]
|
|
@@ -204,7 +204,7 @@ const second = await core.seek(3) // returns [1, 0]
|
|
|
204
204
|
const third = await core.seek(5) // returns [2, 1]
|
|
205
205
|
```
|
|
206
206
|
|
|
207
|
-
```
|
|
207
|
+
```js
|
|
208
208
|
{
|
|
209
209
|
wait: true, // wait for data to be downloaded
|
|
210
210
|
timeout: 0, // wait at max some milliseconds (0 means no timeout)
|
|
@@ -216,7 +216,7 @@ const third = await core.seek(5) // returns [2, 1]
|
|
|
216
216
|
|
|
217
217
|
Make a read stream to read a range of data out at once.
|
|
218
218
|
|
|
219
|
-
```
|
|
219
|
+
```js
|
|
220
220
|
// read the full core
|
|
221
221
|
const fullStream = core.createReadStream()
|
|
222
222
|
|
|
@@ -233,7 +233,7 @@ for await (const data of fullStream) {
|
|
|
233
233
|
|
|
234
234
|
`options` include:
|
|
235
235
|
|
|
236
|
-
```
|
|
236
|
+
```js
|
|
237
237
|
{
|
|
238
238
|
start: 0,
|
|
239
239
|
end: core.length,
|
|
@@ -248,7 +248,7 @@ for await (const data of fullStream) {
|
|
|
248
248
|
|
|
249
249
|
Make a byte stream to read a range of bytes.
|
|
250
250
|
|
|
251
|
-
```
|
|
251
|
+
```js
|
|
252
252
|
// Read the full core
|
|
253
253
|
const fullStream = core.createByteStream()
|
|
254
254
|
|
|
@@ -266,7 +266,7 @@ partialStream.pipe(process.stdout)
|
|
|
266
266
|
|
|
267
267
|
`options` include:
|
|
268
268
|
|
|
269
|
-
```
|
|
269
|
+
```js
|
|
270
270
|
{
|
|
271
271
|
byteOffset: 0, // Offset where to start from
|
|
272
272
|
byteLength: core.byteLength - options.byteOffset, // How many bytes to read
|
|
@@ -278,11 +278,11 @@ partialStream.pipe(process.stdout)
|
|
|
278
278
|
|
|
279
279
|
Make a write stream to append chunks as blocks.
|
|
280
280
|
|
|
281
|
-
```
|
|
281
|
+
```js
|
|
282
282
|
const ws = core.createWriteStream()
|
|
283
283
|
|
|
284
284
|
// Listen for stream finishing
|
|
285
|
-
const done = new Promise(resolve => ws.on('finish', resolve))
|
|
285
|
+
const done = new Promise((resolve) => ws.on('finish', resolve))
|
|
286
286
|
|
|
287
287
|
for (const data of ['hello', 'world']) ws.write(data)
|
|
288
288
|
ws.end()
|
|
@@ -297,7 +297,7 @@ console.log(await core.get(core.length - 1)) // 'world'
|
|
|
297
297
|
|
|
298
298
|
Clear stored blocks between `start` and `end`, reclaiming storage when possible.
|
|
299
299
|
|
|
300
|
-
```
|
|
300
|
+
```js
|
|
301
301
|
await core.clear(4) // clear block 4 from your local cache
|
|
302
302
|
await core.clear(0, 10) // clear block 0-10 from your local cache
|
|
303
303
|
```
|
|
@@ -305,6 +305,7 @@ await core.clear(0, 10) // clear block 0-10 from your local cache
|
|
|
305
305
|
The core will also gossip to peers it is connected to, that is no longer has these blocks.
|
|
306
306
|
|
|
307
307
|
`options` include:
|
|
308
|
+
|
|
308
309
|
```js
|
|
309
310
|
{
|
|
310
311
|
diff: false // Returned `cleared` bytes object is null unless you enable this
|
|
@@ -319,6 +320,7 @@ Per default this will update the fork id of the core to `+ 1`, but you can set t
|
|
|
319
320
|
Note that the fork id should be monotonely incrementing.
|
|
320
321
|
|
|
321
322
|
`options` include:
|
|
323
|
+
|
|
322
324
|
```js
|
|
323
325
|
{
|
|
324
326
|
fork: core.fork + 1, // The new fork id after truncating
|
|
@@ -372,7 +374,7 @@ await range.done()
|
|
|
372
374
|
|
|
373
375
|
A range can have the following properties:
|
|
374
376
|
|
|
375
|
-
```
|
|
377
|
+
```js
|
|
376
378
|
{
|
|
377
379
|
start: startIndex,
|
|
378
380
|
end: nonInclusiveEndIndex,
|
|
@@ -384,7 +386,7 @@ A range can have the following properties:
|
|
|
384
386
|
|
|
385
387
|
To download the full core continuously (often referred to as non sparse mode) do
|
|
386
388
|
|
|
387
|
-
```
|
|
389
|
+
```js
|
|
388
390
|
// Note that this will never be considered downloaded as the range
|
|
389
391
|
// will keep waiting for new blocks to be appended.
|
|
390
392
|
core.download({ start: 0, end: -1 })
|
|
@@ -398,7 +400,7 @@ core.download({ blocks: [4, 9, 7] })
|
|
|
398
400
|
|
|
399
401
|
To cancel downloading a range simply destroy the range instance.
|
|
400
402
|
|
|
401
|
-
```
|
|
403
|
+
```js
|
|
402
404
|
// will stop downloading now
|
|
403
405
|
range.destroy()
|
|
404
406
|
```
|
|
@@ -504,10 +506,10 @@ Info {
|
|
|
504
506
|
fork: 0,
|
|
505
507
|
padding: 8,
|
|
506
508
|
storage: {
|
|
507
|
-
oplog: 8192,
|
|
508
|
-
tree: 4096,
|
|
509
|
-
blocks: 4096,
|
|
510
|
-
bitfield: 4096
|
|
509
|
+
oplog: 8192,
|
|
510
|
+
tree: 4096,
|
|
511
|
+
blocks: 4096,
|
|
512
|
+
bitfield: 4096
|
|
511
513
|
}
|
|
512
514
|
}
|
|
513
515
|
```
|
|
@@ -660,7 +662,7 @@ If you are using a P2P swarm like [Hyperswarm](https://github.com/hyperswarm/hyp
|
|
|
660
662
|
If you want to multiplex the replication over an existing Hypercore replication stream you can pass
|
|
661
663
|
another stream instance instead of the `isInitiator` boolean.
|
|
662
664
|
|
|
663
|
-
```
|
|
665
|
+
```js
|
|
664
666
|
// assuming we have two cores, localCore + remoteCore, sharing the same key
|
|
665
667
|
// on a server
|
|
666
668
|
const net = require('net')
|