hypercore 9.12.0 → 10.0.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/.github/workflows/test-node.yml +3 -4
  2. package/README.md +131 -404
  3. package/__snapshots__/test/storage.js.snapshot.cjs +15 -0
  4. package/examples/announce.js +19 -0
  5. package/examples/basic.js +10 -0
  6. package/examples/http.js +123 -0
  7. package/examples/lookup.js +20 -0
  8. package/index.js +365 -1600
  9. package/lib/bitfield.js +113 -285
  10. package/lib/block-encryption.js +68 -0
  11. package/lib/block-store.js +58 -0
  12. package/lib/core.js +468 -0
  13. package/lib/extensions.js +76 -0
  14. package/lib/merkle-tree.js +1110 -0
  15. package/lib/messages.js +571 -0
  16. package/lib/mutex.js +39 -0
  17. package/lib/oplog.js +224 -0
  18. package/lib/protocol.js +525 -0
  19. package/lib/random-iterator.js +46 -0
  20. package/lib/remote-bitfield.js +24 -0
  21. package/lib/replicator.js +857 -0
  22. package/lib/streams.js +39 -0
  23. package/package.json +44 -45
  24. package/test/basic.js +59 -471
  25. package/test/bitfield.js +48 -133
  26. package/test/core.js +290 -0
  27. package/test/encodings.js +18 -0
  28. package/test/encryption.js +123 -0
  29. package/test/extension.js +71 -0
  30. package/test/helpers/index.js +23 -0
  31. package/test/merkle-tree.js +518 -0
  32. package/test/mutex.js +137 -0
  33. package/test/oplog.js +399 -0
  34. package/test/preload.js +72 -0
  35. package/test/replicate.js +227 -824
  36. package/test/sessions.js +173 -0
  37. package/test/storage.js +31 -0
  38. package/test/streams.js +39 -146
  39. package/test/user-data.js +47 -0
  40. package/bench/all.sh +0 -65
  41. package/bench/copy-64kb-blocks.js +0 -51
  42. package/bench/helpers/read-throttled.js +0 -27
  43. package/bench/helpers/read.js +0 -47
  44. package/bench/helpers/write.js +0 -29
  45. package/bench/read-16kb-blocks-proof-throttled.js +0 -1
  46. package/bench/read-16kb-blocks-proof.js +0 -1
  47. package/bench/read-16kb-blocks-throttled.js +0 -1
  48. package/bench/read-16kb-blocks.js +0 -1
  49. package/bench/read-512b-blocks.js +0 -1
  50. package/bench/read-64kb-blocks-linear-batch.js +0 -18
  51. package/bench/read-64kb-blocks-linear.js +0 -18
  52. package/bench/read-64kb-blocks-proof.js +0 -1
  53. package/bench/read-64kb-blocks.js +0 -1
  54. package/bench/replicate-16kb-blocks.js +0 -19
  55. package/bench/replicate-64kb-blocks.js +0 -19
  56. package/bench/write-16kb-blocks.js +0 -1
  57. package/bench/write-512b-blocks.js +0 -1
  58. package/bench/write-64kb-blocks-static.js +0 -1
  59. package/bench/write-64kb-blocks.js +0 -1
  60. package/example.js +0 -23
  61. package/lib/cache.js +0 -26
  62. package/lib/crypto.js +0 -5
  63. package/lib/replicate.js +0 -829
  64. package/lib/safe-buffer-equals.js +0 -6
  65. package/lib/storage.js +0 -421
  66. package/lib/tree-index.js +0 -183
  67. package/test/ack.js +0 -306
  68. package/test/audit.js +0 -36
  69. package/test/cache.js +0 -93
  70. package/test/compat.js +0 -209
  71. package/test/copy.js +0 -377
  72. package/test/default-storage.js +0 -51
  73. package/test/extensions.js +0 -137
  74. package/test/get.js +0 -64
  75. package/test/head.js +0 -65
  76. package/test/helpers/create-tracking-ram.js +0 -27
  77. package/test/helpers/create.js +0 -6
  78. package/test/helpers/replicate.js +0 -4
  79. package/test/seek.js +0 -234
  80. package/test/selections.js +0 -95
  81. package/test/set-uploading-downloading.js +0 -91
  82. package/test/stats.js +0 -77
  83. package/test/timeouts.js +0 -22
  84. package/test/tree-index.js +0 -841
  85. package/test/update.js +0 -156
  86. package/test/value-encoding.js +0 -52
package/test/head.js DELETED
@@ -1,65 +0,0 @@
1
- var create = require('./helpers/create')
2
- var replicate = require('./helpers/replicate')
3
- var tape = require('tape')
4
-
5
- tape('head without update does not update', t => {
6
- var feed1 = create()
7
- var feed2 = null
8
-
9
- feed1.append('hello', () => {
10
- feed2 = create(feed1.key)
11
- replicate(feed1, feed2, { live: true })
12
- feed2.head((err, content) => {
13
- t.true(err)
14
- t.end()
15
- })
16
- })
17
- })
18
-
19
- tape('head with update waits for an update', t => {
20
- var feed1 = create()
21
- var feed2 = null
22
-
23
- feed1.append('hello', () => {
24
- feed2 = create(feed1.key)
25
- feed2.head({ update: true }, (err, content) => {
26
- t.error(err, 'no error')
27
- t.same(content, Buffer.from('hello'))
28
- t.end()
29
- })
30
- setTimeout(() => {
31
- replicate(feed1, feed2)
32
- }, 50)
33
- })
34
- })
35
-
36
- tape('head with update/ifAvailable will wait only if an update is available', t => {
37
- var feed1 = create()
38
- var feed2 = null
39
-
40
- feed1.append('hello', () => {
41
- feed2 = create(feed1.key)
42
- feed2.head({ update: true, ifAvailable: true }, (err, content) => {
43
- t.error(err, 'no error')
44
- t.same(content, Buffer.from('hello'))
45
- t.end()
46
- })
47
- replicate(feed1, feed2)
48
- })
49
- })
50
-
51
- tape('head with update/ifAvailable will not wait forever', t => {
52
- var feed1 = create()
53
- var feed2 = null
54
-
55
- feed1.append('hello', () => {
56
- feed2 = create(feed1.key)
57
- feed2.head({ update: true, ifAvailable: true }, (err, content) => {
58
- t.true(err)
59
- t.end()
60
- })
61
- setTimeout(() => {
62
- replicate(feed1, feed2)
63
- }, 50)
64
- })
65
- })
@@ -1,27 +0,0 @@
1
- var ram = require('random-access-memory')
2
-
3
- module.exports = function () {
4
- var logByFilename = {}
5
- var factory = function (filename) {
6
- var memory = ram()
7
- var log = []
8
- logByFilename[filename] = log
9
- return {
10
- read: logAndForward('read'),
11
- write: logAndForward('write'),
12
- del: logAndForward('del')
13
- }
14
-
15
- function logAndForward (op) {
16
- return function () {
17
- var statement = {}
18
- statement[op] = [].slice.apply(arguments)
19
- statement[op].pop()
20
- log.push(statement)
21
- return memory[op].apply(memory, arguments)
22
- }
23
- }
24
- }
25
- factory.log = logByFilename
26
- return factory
27
- }
@@ -1,6 +0,0 @@
1
- var hypercore = require('../..')
2
- var ram = require('random-access-memory')
3
-
4
- module.exports = function create (key, opts) {
5
- return hypercore(ram, key, opts)
6
- }
@@ -1,4 +0,0 @@
1
- module.exports = function replicate (a, b, opts, bOpts) {
2
- var stream = a.replicate(false, opts)
3
- return stream.pipe(b.replicate(true, bOpts || opts)).pipe(stream)
4
- }
package/test/seek.js DELETED
@@ -1,234 +0,0 @@
1
- var create = require('./helpers/create')
2
- var tape = require('tape')
3
- var replicate = require('./helpers/replicate')
4
-
5
- tape('seek to byte offset', function (t) {
6
- var feed = create()
7
-
8
- feed.append(['hello', 'how', 'are', 'you', 'doing', '?'])
9
-
10
- feed.flush(function () {
11
- feed.seek(9, function (err, index, offset) {
12
- t.error(err, 'no error')
13
- t.same(index, 2)
14
- t.same(offset, 1)
15
- t.end()
16
- })
17
- })
18
- })
19
-
20
- tape('seek twice', function (t) {
21
- t.plan(6)
22
-
23
- var feed = create()
24
-
25
- feed.append(['hello', 'how', 'are', 'you', 'doing', '?'])
26
-
27
- feed.flush(function () {
28
- feed.seek(9, function (err, index, offset) {
29
- t.error(err, 'no error')
30
- t.same(index, 2)
31
- t.same(offset, 1)
32
- })
33
-
34
- feed.seek(16, function (err, index, offset) {
35
- t.error(err, 'no error')
36
- t.same(index, 4)
37
- t.same(offset, 2)
38
- })
39
- })
40
- })
41
-
42
- tape('seek many times', function (t) {
43
- t.plan(12)
44
-
45
- var feed = create()
46
-
47
- feed.append(['foo', 'b', 'ar', 'baz'], function () {
48
- feed.seek(0, function (err, index, offset) {
49
- t.error(err, 'no error')
50
- t.same(index, 0)
51
- t.same(offset, 0)
52
- })
53
-
54
- feed.seek(2, function (err, index, offset) {
55
- t.error(err, 'no error')
56
- t.same(index, 0)
57
- t.same(offset, 2)
58
- })
59
-
60
- feed.seek(4, function (err, index, offset) {
61
- t.error(err, 'no error')
62
- t.same(index, 2)
63
- t.same(offset, 0)
64
- })
65
-
66
- feed.seek(5, function (err, index, offset) {
67
- t.error(err, 'no error')
68
- t.same(index, 2)
69
- t.same(offset, 1)
70
- })
71
- })
72
- })
73
-
74
- tape('seek waits', function (t) {
75
- t.plan(6)
76
-
77
- var feed = create()
78
-
79
- feed.seek(9, function (err, index, offset) {
80
- t.error(err, 'no error')
81
- t.same(index, 2)
82
- t.same(offset, 1)
83
- })
84
-
85
- feed.seek(16, function (err, index, offset) {
86
- t.error(err, 'no error')
87
- t.same(index, 4)
88
- t.same(offset, 2)
89
- })
90
-
91
- feed.append(['hello'], function () {
92
- feed.append(['how', 'are', 'you', 'doing', '?'])
93
- })
94
- })
95
-
96
- tape('seek works for sparse trees', function (t) {
97
- var feed = create()
98
-
99
- feed.append('aa', function () {
100
- var clone = create(feed.key, { sparse: true })
101
-
102
- replicate(feed, clone, { live: true })
103
-
104
- clone.get(0, function () { // make sure we have a tree rooted at 0
105
- const chunks = Array(15)
106
- chunks.fill('aa')
107
- feed.append(chunks, function () {
108
- clone.get(15, function () { // get an updated tree that is disconnected with the prev one
109
- clone.seek(1, function (err, index, offset) { // old seek still works
110
- t.error(err, 'no error')
111
- t.same(index, 0)
112
- t.same(offset, 1)
113
- clone.seek(8, function (err, index, offset) {
114
- t.error(err, 'no error')
115
- t.same(index, 4)
116
- t.same(offset, 0)
117
- t.end()
118
- })
119
- })
120
- })
121
- })
122
- })
123
- })
124
- })
125
-
126
- tape('seek to sibling', function (t) {
127
- t.plan(9)
128
-
129
- var feed = create()
130
-
131
- feed.append(['aa', 'aa'], function () {
132
- feed.seek(2, function (err, index, offset) { // sibling seek
133
- t.error(err, 'no error')
134
- t.same(index, 1)
135
- t.same(offset, 0)
136
- })
137
- feed.seek(3, function (err, index, offset) {
138
- t.error(err, 'no error')
139
- t.same(index, 1)
140
- t.same(offset, 1)
141
- })
142
- feed.seek(1, function (err, index, offset) {
143
- t.error(err, 'no error')
144
- t.same(index, 0)
145
- t.same(offset, 1)
146
- })
147
- })
148
- })
149
-
150
- tape('seek to 0 and byteLength', function (t) {
151
- t.plan(6)
152
-
153
- var feed = create()
154
-
155
- feed.append(['a', 'b', 'c'], function () {
156
- feed.seek(0, function (err, index, offset) {
157
- t.same(err, null)
158
- t.same(index, 0)
159
- t.same(offset, 0)
160
- })
161
-
162
- feed.seek(feed.byteLength, function (err, index, offset) {
163
- t.same(err, null)
164
- t.same(index, feed.length)
165
- t.same(offset, 0)
166
- })
167
- })
168
- })
169
-
170
- tape('seek ifAvailable', function (t) {
171
- var feed = create()
172
-
173
- feed.append(['a', 'b', 'c'], function () {
174
- var clone = create(feed.key, { sparse: true })
175
-
176
- replicate(feed, clone, { live: true })
177
-
178
- clone.seek(4, { ifAvailable: true }, function (err) {
179
- t.ok(err, 'should error')
180
- clone.seek(2, { ifAvailable: true }, function (err, index, offset) {
181
- t.error(err, 'no error')
182
- t.same(index, 2)
183
- t.same(offset, 0)
184
- t.end()
185
- })
186
- })
187
- })
188
- })
189
-
190
- tape('seek ifAvailable multiple peers', function (t) {
191
- var feed = create()
192
-
193
- feed.append(['a', 'b', 'c'], function () {
194
- var clone1 = create(feed.key, { sparse: true })
195
- var clone2 = create(feed.key, { sparse: true })
196
-
197
- replicate(feed, clone1, { live: true })
198
- replicate(clone1, clone2, { live: true })
199
-
200
- clone2.seek(2, { ifAvailable: true }, function (err) {
201
- t.ok(err, 'should error')
202
- clone1.get(2, function () {
203
- clone2.seek(2, { ifAvailable: true }, function (err, index, offset) {
204
- t.error(err, 'no error')
205
- t.same(index, 2)
206
- t.same(offset, 0)
207
- t.end()
208
- })
209
- })
210
- })
211
- })
212
- })
213
-
214
- tape('seek ifAvailable with many inflight requests', function (t) {
215
- var feed = create()
216
-
217
- var arr = new Array(100).fill('a')
218
-
219
- feed.append(arr, function () {
220
- var clone = create(feed.key, { sparse: true })
221
-
222
- replicate(feed, clone, { live: true })
223
-
224
- // Create 100 inflight requests.
225
- for (let i = 0; i < 100; i++) clone.get(i, () => {})
226
-
227
- clone.seek(2, { ifAvailable: true }, function (err, index, offset) {
228
- t.error(err, 'no error')
229
- t.same(index, 2)
230
- t.same(offset, 0)
231
- t.end()
232
- })
233
- })
234
- })
@@ -1,95 +0,0 @@
1
- var create = require('./helpers/create')
2
- var tape = require('tape')
3
-
4
- tape('cancel', function (t) {
5
- t.plan(2)
6
-
7
- var feed = create()
8
-
9
- feed.get(0, function (err) {
10
- t.ok(err, 'expected error')
11
- })
12
-
13
- feed.get(0, function (err) {
14
- t.ok(err, 'expected error')
15
- })
16
-
17
- feed.cancel(0)
18
- })
19
-
20
- tape('cancel range', function (t) {
21
- t.plan(2)
22
-
23
- var feed = create()
24
-
25
- feed.get(0, function (err) {
26
- t.ok(err, 'expected error')
27
- })
28
-
29
- feed.get(1, function (err) {
30
- t.ok(err, 'expected error')
31
- })
32
-
33
- feed.get(2, function () {
34
- t.fail('should not error')
35
- })
36
-
37
- feed.cancel(0, 2)
38
- })
39
-
40
- tape('get after cancel', function (t) {
41
- t.plan(1)
42
-
43
- var feed = create()
44
-
45
- feed.get(0, function (err) {
46
- t.ok(err, 'expected error')
47
- feed.get(0, function () {
48
- t.fail('should not error')
49
- })
50
- })
51
-
52
- feed.cancel(0)
53
- })
54
-
55
- tape('cancel download', function (t) {
56
- var feed = create()
57
-
58
- feed.download({ start: 0, end: 10 }, function (err) {
59
- t.ok(err, 'expected error')
60
- t.end()
61
- })
62
-
63
- feed.cancel(0, 10)
64
- })
65
-
66
- tape('cancel download and get', function (t) {
67
- t.plan(3)
68
-
69
- var feed = create()
70
-
71
- feed.download({ start: 1, end: 9 }, function (err) {
72
- t.ok(err, 'expected error')
73
- })
74
-
75
- feed.get(5, function (err) {
76
- t.ok(err, 'expected error')
77
- })
78
-
79
- feed.get(7, function (err) {
80
- t.ok(err, 'expected error')
81
- })
82
-
83
- feed.cancel(0, 10)
84
- })
85
-
86
- tape('cancel seek', function (t) {
87
- var feed = create()
88
-
89
- feed.seek(10, { start: 0, end: 10 }, function (err) {
90
- t.ok(err, 'expected error')
91
- t.end()
92
- })
93
-
94
- feed.cancel(0, 10)
95
- })
@@ -1,91 +0,0 @@
1
- const tape = require('tape')
2
- const create = require('./helpers/create')
3
- const replicate = require('./helpers/replicate')
4
-
5
- tape('setDownloading', function (t) {
6
- const feed = create()
7
-
8
- feed.append(['a', 'b'], function () {
9
- const clone = create(feed.key)
10
-
11
- clone.setDownloading(false)
12
- let later = false
13
-
14
- clone.get(0, function (err, data) {
15
- t.error(err, 'no error')
16
- t.same(data, Buffer.from('a'))
17
- t.ok(later)
18
-
19
- clone.ifAvailable.ready(function () {
20
- feed.ifAvailable.ready(function () {
21
- t.pass('if available drained')
22
- t.end()
23
- })
24
- })
25
- })
26
-
27
- replicate(feed, clone, { live: true })
28
-
29
- setImmediate(() => {
30
- later = true
31
- clone.setDownloading(true)
32
- })
33
- })
34
- })
35
-
36
- tape('setUploading', function (t) {
37
- const feed = create()
38
-
39
- feed.append(['a', 'b'], function () {
40
- const clone = create(feed.key)
41
-
42
- feed.setUploading(false)
43
- let later = false
44
-
45
- clone.get(0, function (err, data) {
46
- t.error(err, 'no error')
47
- t.same(data, Buffer.from('a'))
48
- t.ok(later)
49
- clone.ifAvailable.ready(function () {
50
- feed.ifAvailable.ready(function () {
51
- t.pass('if available drained')
52
- t.end()
53
- })
54
- })
55
- })
56
-
57
- replicate(feed, clone)
58
-
59
- setImmediate(() => {
60
- later = true
61
- feed.setUploading(true)
62
- })
63
- })
64
- })
65
-
66
- tape('get block while not uploading', function (t) {
67
- const feed = create()
68
-
69
- feed.append(['a', 'b', 'c', 'd'], function () {
70
- const clone = create(feed.key, { sparse: true })
71
-
72
- clone.get(0, function (err, data) {
73
- t.error(err, 'no error')
74
- t.same(data, Buffer.from('a'))
75
- feed.setUploading(false)
76
- clone.get(3, function (err, data) {
77
- t.ok(feed.uploading)
78
- t.ok(clone.peers[0].remoteUploading)
79
- t.error(err, 'no error')
80
- t.same(data, Buffer.from('d'))
81
- t.end()
82
- })
83
-
84
- setImmediate(function () {
85
- feed.setUploading(true)
86
- })
87
- })
88
-
89
- replicate(feed, clone)
90
- })
91
- })
package/test/stats.js DELETED
@@ -1,77 +0,0 @@
1
- var create = require('./helpers/create')
2
- var replicate = require('./helpers/replicate')
3
- var tape = require('tape')
4
-
5
- tape('accurate stat totals', function (t) {
6
- t.plan(4)
7
-
8
- var feed = create()
9
- feed.append(['aa', 'bb', 'cc', 'dd', 'ee'], function () {
10
- var clone = create(feed.key)
11
- replicate(feed, clone).on('end', function () {
12
- var feedStats = feed.stats
13
- var cloneStats = clone.stats
14
-
15
- t.same(feedStats.totals.uploadedBlocks, 5)
16
- t.same(feedStats.totals.uploadedBytes, 10)
17
- t.same(cloneStats.totals.downloadedBlocks, 5)
18
- t.same(cloneStats.totals.downloadedBytes, 10)
19
- })
20
- })
21
- })
22
-
23
- tape('accurate per-peer stats', function (t) {
24
- t.plan(13)
25
-
26
- var feed = create()
27
-
28
- feed.append(['aa', 'bb', 'cc', 'dd', 'ee'], function () {
29
- var clone1 = create(feed.key)
30
- var clone2 = create(feed.key)
31
-
32
- replicate(feed, clone1, { live: true })
33
- replicate(feed, clone2, { live: true })
34
-
35
- setTimeout(function () {
36
- onreplicate(clone1, clone2)
37
- }, 50)
38
- })
39
-
40
- function onreplicate (clone1, clone2) {
41
- var feedStats = feed.stats
42
- var clone1Stats = clone1.stats
43
- var clone2Stats = clone2.stats
44
-
45
- t.same(feedStats.totals.uploadedBlocks, 10)
46
- t.same(feedStats.totals.uploadedBytes, 20)
47
- t.same(feedStats.peers.length, 2)
48
- t.same(feedStats.peers[0].uploadedBlocks, 5)
49
- t.same(feedStats.peers[0].uploadedBytes, 10)
50
- t.same(feedStats.peers[1].uploadedBlocks, 5)
51
- t.same(feedStats.peers[1].uploadedBytes, 10)
52
-
53
- t.same(clone1Stats.peers.length, 1)
54
- t.same(clone1Stats.peers[0].downloadedBytes, 10)
55
- t.same(clone1Stats.peers[0].downloadedBlocks, 5)
56
-
57
- t.same(clone2Stats.peers.length, 1)
58
- t.same(clone2Stats.peers[0].downloadedBytes, 10)
59
- t.same(clone2Stats.peers[0].downloadedBlocks, 5)
60
- }
61
- })
62
-
63
- tape('should not collect stats when stats option is false', function (t) {
64
- t.plan(2)
65
-
66
- var feed = create({ stats: false })
67
- feed.append(['aa', 'bb', 'cc', 'dd', 'ee'], function () {
68
- var clone = create(feed.key, { stats: false })
69
- replicate(feed, clone).on('end', function () {
70
- var feedStats = feed.stats
71
- var cloneStats = clone.stats
72
-
73
- t.false(feedStats)
74
- t.false(cloneStats)
75
- })
76
- })
77
- })
package/test/timeouts.js DELETED
@@ -1,22 +0,0 @@
1
- const tape = require('tape')
2
- const create = require('./helpers/create')
3
-
4
- tape('get before timeout', function (t) {
5
- const feed = create()
6
-
7
- feed.get(0, { timeout: 100 }, function (err) {
8
- t.error(err, 'no timeout error')
9
- t.end()
10
- })
11
-
12
- feed.append('hi')
13
- })
14
-
15
- tape('get after timeout', function (t) {
16
- const feed = create()
17
-
18
- feed.get(42, { timeout: 100 }, function (err) {
19
- t.ok(err, 'had timeout')
20
- t.end()
21
- })
22
- })