autobee 2.10.1 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +129 -5
- package/lib/constants.js +2 -0
- package/lib/encoding.js +2 -2
- package/lib/encryption.js +259 -0
- package/lib/fast-forward.js +36 -14
- package/lib/migrations.js +1 -1
- package/lib/writers.js +1 -1
- package/package.json +5 -5
package/index.js
CHANGED
|
@@ -5,15 +5,17 @@ const safetyCatch = require('safety-catch')
|
|
|
5
5
|
const Hyperbee = require('hyperbee2')
|
|
6
6
|
const ID = require('hypercore-id-encoding')
|
|
7
7
|
const rrp = require('resolve-reject-promise')
|
|
8
|
-
const
|
|
8
|
+
const Signal = require('signal-promise')
|
|
9
9
|
const AutobeeWakeup = require('autobee-wakeup')
|
|
10
10
|
const Hypercore = require('hypercore')
|
|
11
11
|
const crypto = require('hypercore-crypto')
|
|
12
12
|
const c = require('compact-encoding')
|
|
13
13
|
const asserts = require('./lib/asserts.js')
|
|
14
14
|
const boot = require('./lib/boot.js')
|
|
15
|
+
const { DEFAULT_MANIFEST_VERSION } = require('./lib/constants.js')
|
|
15
16
|
const { resolveWeight, currentWeight } = require('./lib/witness.js')
|
|
16
17
|
const encoding = require('./lib/encoding.js')
|
|
18
|
+
const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('./lib/encryption.js')
|
|
17
19
|
const FastForward = require('./lib/fast-forward.js')
|
|
18
20
|
const System = require('./lib/system.js')
|
|
19
21
|
const ApplyCalls = require('./lib/apply-calls.js')
|
|
@@ -26,6 +28,7 @@ const migrations = require('./lib/migrations.js')
|
|
|
26
28
|
|
|
27
29
|
const EMPTY_HEAD = { length: 0, key: null }
|
|
28
30
|
const DEFAULT_ACK_THRESHOLD = 32
|
|
31
|
+
const BOOT_NETWORK_TIMEOUT = 5000
|
|
29
32
|
const INTERRUPT = new Error('Apply interrupted')
|
|
30
33
|
|
|
31
34
|
module.exports = class Autobee extends ReadyResource {
|
|
@@ -116,6 +119,9 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
116
119
|
this.fastForwardTo = null
|
|
117
120
|
this._ffCandidates = new Map()
|
|
118
121
|
this._ffSearching = null
|
|
122
|
+
this._bumpSignal = new Signal()
|
|
123
|
+
this._networkBooted = false
|
|
124
|
+
this._networkBooting = false
|
|
119
125
|
|
|
120
126
|
this._workingBee = bee
|
|
121
127
|
this._workingView = new ApplyView(this._workingBee, this)
|
|
@@ -143,6 +149,7 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
143
149
|
this._now = handlers.now || Date.now // overridable for clock-drift tests
|
|
144
150
|
this._preapply = handlers.preapply || null
|
|
145
151
|
this._preApplied = false
|
|
152
|
+
this._reindexed = false
|
|
146
153
|
this._warmup = handlers.warmup || null
|
|
147
154
|
this._hasApply = !!handlers.apply
|
|
148
155
|
this._hasUpdate = !!handlers.update
|
|
@@ -205,7 +212,8 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
205
212
|
await 1
|
|
206
213
|
const result = await this._prebooting
|
|
207
214
|
return {
|
|
208
|
-
name: 'autobee/' + result.local.id + '/' + name,
|
|
215
|
+
name: 'autobee/' + result.local.id + '/view/' + name,
|
|
216
|
+
manifestVersion: DEFAULT_MANIFEST_VERSION,
|
|
209
217
|
encryption: name === 'system' ? this.getSystemEncryption() : this.getViewEncryption(),
|
|
210
218
|
inflightRange: [256, 512]
|
|
211
219
|
}
|
|
@@ -380,6 +388,7 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
380
388
|
|
|
381
389
|
async _close() {
|
|
382
390
|
this._interrupting = true
|
|
391
|
+
this._bumpSignal.notify()
|
|
383
392
|
|
|
384
393
|
// the local core holds the exclusive lock but the local writer closes it
|
|
385
394
|
// early in the teardown, so hand the lock to a detached session that
|
|
@@ -583,6 +592,11 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
583
592
|
this._localFlushes = this.system.flushes
|
|
584
593
|
}
|
|
585
594
|
|
|
595
|
+
async _isReindexing() {
|
|
596
|
+
if (this._networkBooted || !this._ffEnabled || this._migrating) return false
|
|
597
|
+
return !(await this._isReindexed())
|
|
598
|
+
}
|
|
599
|
+
|
|
586
600
|
async _bootOnline() {
|
|
587
601
|
if (!this._bootOnlineGuard.enter()) return
|
|
588
602
|
|
|
@@ -607,6 +621,7 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
607
621
|
}
|
|
608
622
|
|
|
609
623
|
bumpSoon() {
|
|
624
|
+
this._bumpSignal.notify()
|
|
610
625
|
this._bump(false).catch(safetyCatch)
|
|
611
626
|
}
|
|
612
627
|
|
|
@@ -668,6 +683,61 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
668
683
|
this.emit('error', err)
|
|
669
684
|
}
|
|
670
685
|
|
|
686
|
+
async compactMaybe() {
|
|
687
|
+
if (!this.writers.writable) return
|
|
688
|
+
|
|
689
|
+
if (await this._isReindexed()) return
|
|
690
|
+
|
|
691
|
+
await this._reindexBee(this._workingBee)
|
|
692
|
+
this.bee.move(this._workingBee.head())
|
|
693
|
+
|
|
694
|
+
await this._reindexBee(this.system.bee)
|
|
695
|
+
|
|
696
|
+
this._reindexed = true
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
async _isReindexed() {
|
|
700
|
+
const head = this.system.bee.head()
|
|
701
|
+
if (head === null) return true
|
|
702
|
+
if (await this._shouldReindex(head.key)) return false
|
|
703
|
+
|
|
704
|
+
const view = this.system.view
|
|
705
|
+
if (!view || !view.key) return true
|
|
706
|
+
return !(await this._shouldReindex(view.key))
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
async _reindexBee(bee) {
|
|
710
|
+
const head = bee.head()
|
|
711
|
+
const local = bee.context.local
|
|
712
|
+
if (head === null) return
|
|
713
|
+
if (await this._shouldReindex(local.key)) return
|
|
714
|
+
if (!(await this._shouldReindex(head.key))) return
|
|
715
|
+
|
|
716
|
+
// a non-empty local core is a completed reindex whose head was never stored
|
|
717
|
+
if (local.length === 0) {
|
|
718
|
+
await bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
bee.move({ key: local.key, length: local.length })
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
async _shouldReindex(key, { unknown = false, length = 0, timeout = 0 } = {}) {
|
|
725
|
+
const core = this.store.get({ key, active: false })
|
|
726
|
+
|
|
727
|
+
try {
|
|
728
|
+
await core.ready()
|
|
729
|
+
|
|
730
|
+
if (core.manifest === null && length > 0) {
|
|
731
|
+
await core.get(length - 1, { raw: true, timeout }).catch(safetyCatch)
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
if (core.manifest === null) return unknown
|
|
735
|
+
return core.manifest.version > 1 && core.manifest.version < DEFAULT_MANIFEST_VERSION
|
|
736
|
+
} finally {
|
|
737
|
+
await core.close()
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
671
741
|
// one-shot user gate: nothing applies until the host has resolved whatever
|
|
672
742
|
// state apply depends on (e.g. legacy views recorded by a migration)
|
|
673
743
|
async _runPreApply() {
|
|
@@ -688,6 +758,7 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
688
758
|
const { head = null, legacy = null } = this.bootFrom
|
|
689
759
|
|
|
690
760
|
this.bootFrom = null
|
|
761
|
+
this._networkBooted = true
|
|
691
762
|
|
|
692
763
|
if (legacy) {
|
|
693
764
|
await this._bootFromSystem(legacy)
|
|
@@ -695,8 +766,14 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
695
766
|
this._wakeup.hint({ key: head.key, length: head.length || 0 })
|
|
696
767
|
await this._bootFromHead(head)
|
|
697
768
|
}
|
|
769
|
+
} else if (await this._isReindexing()) {
|
|
770
|
+
this._networkBooted = true
|
|
771
|
+
await this._bootFromNetwork()
|
|
698
772
|
}
|
|
699
773
|
|
|
774
|
+
// preferably get a peer's compacted view during bootFrom
|
|
775
|
+
if (this.fastForwardTo === null) await this.compactMaybe()
|
|
776
|
+
|
|
700
777
|
const changes = this._hasUpdate ? new UpdateChanges(this) : null
|
|
701
778
|
if (changes) changes.track()
|
|
702
779
|
|
|
@@ -865,6 +942,7 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
865
942
|
const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
|
|
866
943
|
|
|
867
944
|
const ff = await FastForward.fromHeads(this, heads, {
|
|
945
|
+
skipGap: this._networkBooting,
|
|
868
946
|
timeout: FastForward.DEFAULT_TIMEOUT
|
|
869
947
|
})
|
|
870
948
|
|
|
@@ -1208,12 +1286,15 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
1208
1286
|
}
|
|
1209
1287
|
|
|
1210
1288
|
async _appendAck() {
|
|
1211
|
-
if (!this._acking) return false
|
|
1289
|
+
if (!this._acking && !this._reindexed) return false
|
|
1212
1290
|
if (!this.writers.writable) return false
|
|
1213
1291
|
|
|
1214
1292
|
if (this.writers.localWriter.pending !== null) return false
|
|
1215
|
-
|
|
1216
|
-
if (
|
|
1293
|
+
|
|
1294
|
+
if (!this._reindexed) {
|
|
1295
|
+
if ((await this._flushesBehind()) < this._ackThreshold) return false
|
|
1296
|
+
if (await this._allHeadsTrusted()) return false
|
|
1297
|
+
}
|
|
1217
1298
|
|
|
1218
1299
|
const links = this.system.getLinks(this.local.key)
|
|
1219
1300
|
const now = this._now()
|
|
@@ -1371,6 +1452,14 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
1371
1452
|
|
|
1372
1453
|
const changed = await this.system.flush(batch, this._workingBee)
|
|
1373
1454
|
|
|
1455
|
+
if (this._reindexed && (await this._isReindexed())) {
|
|
1456
|
+
this._reindexed = false
|
|
1457
|
+
await this.local.setUserData(
|
|
1458
|
+
'autobee/head',
|
|
1459
|
+
encoding.encodeBootRecord(this.system.bootRecord())
|
|
1460
|
+
)
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1374
1463
|
if (this.system.promotions.changed) this._prefetchApprovals().catch(safetyCatch)
|
|
1375
1464
|
|
|
1376
1465
|
if (local) {
|
|
@@ -1551,6 +1640,41 @@ module.exports = class Autobee extends ReadyResource {
|
|
|
1551
1640
|
}
|
|
1552
1641
|
}
|
|
1553
1642
|
|
|
1643
|
+
async _bootFromNetwork() {
|
|
1644
|
+
this._networkBooting = true
|
|
1645
|
+
|
|
1646
|
+
try {
|
|
1647
|
+
await this._waitForNetworkBoot()
|
|
1648
|
+
} finally {
|
|
1649
|
+
this._networkBooting = false
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
async _waitForNetworkBoot() {
|
|
1654
|
+
const deadline = Date.now() + BOOT_NETWORK_TIMEOUT
|
|
1655
|
+
|
|
1656
|
+
this._requestWakeup()
|
|
1657
|
+
|
|
1658
|
+
while (!this._interrupting && !this.closing && this.fastForwardTo === null) {
|
|
1659
|
+
const hints = await this._applyWakeupHints()
|
|
1660
|
+
if (hints.length) this._queueFastForward(hints)
|
|
1661
|
+
|
|
1662
|
+
const remaining = deadline - Date.now()
|
|
1663
|
+
if (remaining <= 0 || this.fastForwardTo !== null) return
|
|
1664
|
+
|
|
1665
|
+
if (this._ffSearching !== null) {
|
|
1666
|
+
await Promise.race([this._ffSearching, this._bumpSignal.wait(remaining)])
|
|
1667
|
+
this._bumpSignal.notify()
|
|
1668
|
+
if (this._ffSearching === null) return
|
|
1669
|
+
continue
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
if (hints.length) return
|
|
1673
|
+
|
|
1674
|
+
await this._bumpSignal.wait(remaining)
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1554
1678
|
async _runFastForward(ff, { force = false } = {}) {
|
|
1555
1679
|
if (this.fastForwardTo !== null || this.fastForwarding !== null) {
|
|
1556
1680
|
await ff.close()
|
package/lib/constants.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
const LEGACY_OPLOG_VERSION = 2
|
|
2
2
|
const OPLOG_VERSION = 4
|
|
3
|
+
const DEFAULT_MANIFEST_VERSION = 3
|
|
3
4
|
const LEGACY_AUTOBASE_VERSION = 2
|
|
4
5
|
const AUTOBEE_VERSION = 3
|
|
5
6
|
const DEFAULT_OP_TIMEOUT = 5000
|
|
6
7
|
|
|
7
8
|
module.exports = {
|
|
9
|
+
DEFAULT_MANIFEST_VERSION,
|
|
8
10
|
LEGACY_OPLOG_VERSION,
|
|
9
11
|
OPLOG_VERSION,
|
|
10
12
|
LEGACY_AUTOBASE_VERSION,
|
package/lib/encoding.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
const b4a = require('b4a')
|
|
2
2
|
const c = require('compact-encoding')
|
|
3
3
|
const crypto = require('hypercore-crypto')
|
|
4
|
-
const { AutobeeEncryption, WriterEncryption } = require('autobee-encryption')
|
|
5
4
|
|
|
6
5
|
const { getEncoding } = require('../encoding/spec/autobee')
|
|
7
|
-
const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants')
|
|
6
|
+
const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants.js')
|
|
7
|
+
const { AutobeeEncryption, WriterEncryption } = require('./encryption.js')
|
|
8
8
|
|
|
9
9
|
const Oplog = getEncoding('@autobee/oplog')
|
|
10
10
|
const Link = getEncoding('@autobee/link')
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
const HypercoreEncryption = require('hypercore/lib/default-encryption.js')
|
|
2
|
+
const sodium = require('sodium-native')
|
|
3
|
+
const crypto = require('hypercore-crypto')
|
|
4
|
+
const c = require('compact-encoding')
|
|
5
|
+
const b4a = require('b4a')
|
|
6
|
+
|
|
7
|
+
const { DEFAULT_MANIFEST_VERSION } = require('./constants.js')
|
|
8
|
+
|
|
9
|
+
const ManifestData = {
|
|
10
|
+
preencode(state, m) {
|
|
11
|
+
c.uint.preencode(state, m.version)
|
|
12
|
+
state.end++ // max flag is 2 so always one byte
|
|
13
|
+
|
|
14
|
+
if (m.legacyBlocks) c.uint.preencode(state, m.legacyBlocks)
|
|
15
|
+
if (m.namespace) c.fixed32.preencode(state, m.namespace)
|
|
16
|
+
},
|
|
17
|
+
encode(state, m) {
|
|
18
|
+
const flags = (m.legacyBlocks ? 1 : 0) | (m.namespace ? 2 : 0)
|
|
19
|
+
|
|
20
|
+
c.uint.encode(state, m.version)
|
|
21
|
+
c.uint.encode(state, flags)
|
|
22
|
+
|
|
23
|
+
if (m.legacyBlocks) c.uint.encode(state, m.legacyBlocks)
|
|
24
|
+
if (m.namespace) c.fixed32.encode(state, m.namespace)
|
|
25
|
+
},
|
|
26
|
+
decode(state) {
|
|
27
|
+
const r0 = c.uint.decode(state)
|
|
28
|
+
const flags = c.uint.decode(state)
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
version: r0,
|
|
32
|
+
legacyBlocks: (flags & 1) !== 0 ? c.uint.decode(state) : 0,
|
|
33
|
+
namespace: (flags & 2) !== 0 ? c.fixed32.decode(state) : null
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const [, NS_VIEW_BLOCK_KEY, NS_HASH_KEY] = crypto.namespace('autobase', 4)
|
|
39
|
+
|
|
40
|
+
const [GENESIS_ENTROPY] = crypto.namespace('autobase/entropy', 2)
|
|
41
|
+
|
|
42
|
+
const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
|
|
43
|
+
const hash = nonce.subarray(0, sodium.crypto_generichash_BYTES_MIN)
|
|
44
|
+
|
|
45
|
+
class AutobeeEncryption {
|
|
46
|
+
static PADDING = 8
|
|
47
|
+
|
|
48
|
+
constructor(auto) {
|
|
49
|
+
this.auto = auto
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
padding() {
|
|
53
|
+
return AutobeeEncryption.PADDING
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
isCompat(ctx) {
|
|
57
|
+
return ctx.manifest.version <= 1
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async getKeys(id, ctx) {
|
|
61
|
+
if (!this.auto.encryptionKey) return null
|
|
62
|
+
|
|
63
|
+
const entropy = GENESIS_ENTROPY
|
|
64
|
+
const block = this.blockKey(entropy, ctx)
|
|
65
|
+
const hash = crypto.hash([NS_HASH_KEY, block])
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
id,
|
|
69
|
+
block,
|
|
70
|
+
hash
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
blockKey(entropy, ctx) {
|
|
75
|
+
return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
compatKeys() {
|
|
79
|
+
throw new Error('Compatibility method is not specified')
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async encrypt(index, block, fork, ctx) {
|
|
83
|
+
// writer cores are long lasting, but view cores should be up to date
|
|
84
|
+
if (this instanceof ViewEncryption && ctx.manifest.version < DEFAULT_MANIFEST_VERSION) {
|
|
85
|
+
throw new Error(`Unexpected manifest: version < ${DEFAULT_MANIFEST_VERSION}`)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (this.isCompat(ctx, index)) {
|
|
89
|
+
const compat = this.compatKeys(ctx)
|
|
90
|
+
return HypercoreEncryption.encrypt(index, block, fork, compat.block, compat.blinding)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const keys = await this.getKeys(0, ctx)
|
|
94
|
+
|
|
95
|
+
encryptBlock(index, block, keys.id, keys.block, keys.hash)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async decrypt(index, block, ctx) {
|
|
99
|
+
if (this.isCompat(ctx, index)) {
|
|
100
|
+
return HypercoreEncryption.decrypt(index, block, this.compatKeys(ctx).block)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const padding = block.subarray(0, AutobeeEncryption.PADDING)
|
|
104
|
+
block = block.subarray(AutobeeEncryption.PADDING)
|
|
105
|
+
|
|
106
|
+
const type = padding[0]
|
|
107
|
+
switch (type) {
|
|
108
|
+
case 0:
|
|
109
|
+
return block // unencrypted
|
|
110
|
+
|
|
111
|
+
case 1:
|
|
112
|
+
break
|
|
113
|
+
|
|
114
|
+
default:
|
|
115
|
+
throw new Error('Unrecognised encryption type')
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const id = c.uint32.decode({ start: 4, end: 8, buffer: padding })
|
|
119
|
+
|
|
120
|
+
const keys = await this.getKeys(id, ctx)
|
|
121
|
+
|
|
122
|
+
c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
|
|
123
|
+
nonce.set(padding, 8, 16)
|
|
124
|
+
|
|
125
|
+
// Decrypt the block using the full nonce
|
|
126
|
+
decrypt(block, nonce, keys.block)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
static encryptAnchor(block, bootstrap, encryptionKey, namespace) {
|
|
130
|
+
const entropy = GENESIS_ENTROPY
|
|
131
|
+
|
|
132
|
+
const blockKey = getBlockKey(bootstrap, encryptionKey, entropy, namespace)
|
|
133
|
+
const hashKey = crypto.hash([NS_HASH_KEY, block])
|
|
134
|
+
|
|
135
|
+
encryptBlock(0, block, 0, blockKey, hashKey)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
static async setSystemEncryption(bootstrap, encryptionKey, core, opts) {
|
|
139
|
+
await core.ready()
|
|
140
|
+
if (core.manifest.version === 1) {
|
|
141
|
+
const key = getCompatBlockKey(bootstrap, encryptionKey, '_system')
|
|
142
|
+
core.setEncryptionKey(key, { block: true })
|
|
143
|
+
return
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const info = {
|
|
147
|
+
key: bootstrap,
|
|
148
|
+
encryptionKey
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return core.setEncryption(new ViewEncryption(info, '_system'))
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
static getSystemEncryption(bootstrap, encryptionKey) {
|
|
155
|
+
return AutobeeEncryption.getViewEncryption(bootstrap, encryptionKey, '_system')
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
static getViewEncryption(bootstrap, encryptionKey, name) {
|
|
159
|
+
const info = { key: bootstrap, encryptionKey }
|
|
160
|
+
return new ViewEncryption(info, name)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
static getWriterEncryption(bootstrap, encryptionKey) {
|
|
164
|
+
return new WriterEncryption({ key: bootstrap, encryptionKey })
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
class ViewEncryption extends AutobeeEncryption {
|
|
169
|
+
constructor(auto, name) {
|
|
170
|
+
super(auto)
|
|
171
|
+
this.name = name
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
compatKeys() {
|
|
175
|
+
const block = getCompatBlockKey(this.auto.key, this.auto.encryptionKey, this.name)
|
|
176
|
+
return {
|
|
177
|
+
block,
|
|
178
|
+
blinding: crypto.hash(block)
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
blockKey(entropy, ctx) {
|
|
183
|
+
// compat encryption
|
|
184
|
+
if (ctx.manifest.version < DEFAULT_MANIFEST_VERSION) {
|
|
185
|
+
return getCompatBlockKey(this.auto.key, entropy, this.name)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
class WriterEncryption extends AutobeeEncryption {
|
|
193
|
+
compatKeys(ctx) {
|
|
194
|
+
return HypercoreEncryption.deriveKeys(this.auto.encryptionKey, ctx.key)
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
blockKey(entropy, ctx) {
|
|
198
|
+
// used for anchors
|
|
199
|
+
if (ctx.manifest.userData) {
|
|
200
|
+
const userData = c.decode(ManifestData, ctx.manifest.userData)
|
|
201
|
+
if (userData.namespace !== null) {
|
|
202
|
+
const b = getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, userData.namespace)
|
|
203
|
+
return b
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
module.exports = {
|
|
212
|
+
AutobeeEncryption,
|
|
213
|
+
WriterEncryption,
|
|
214
|
+
ViewEncryption
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function encrypt(block, nonce, key) {
|
|
218
|
+
sodium.crypto_stream_xor(block, block, nonce, key)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function decrypt(block, nonce, key) {
|
|
222
|
+
return encrypt(block, nonce, key) // symmetric
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function getBlockKey(bootstrap, encryptionKey, entropy, hypercoreKey) {
|
|
226
|
+
return (
|
|
227
|
+
encryptionKey &&
|
|
228
|
+
crypto.hash([NS_VIEW_BLOCK_KEY, bootstrap, encryptionKey, entropy, hypercoreKey])
|
|
229
|
+
)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function getCompatBlockKey(bootstrap, encryptionKey, name) {
|
|
233
|
+
if (typeof name === 'string') return getCompatBlockKey(bootstrap, encryptionKey, b4a.from(name))
|
|
234
|
+
return encryptionKey && crypto.hash([NS_VIEW_BLOCK_KEY, bootstrap, encryptionKey, name])
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function blockhash(block, padding, hashKey) {
|
|
238
|
+
sodium.crypto_generichash(hash, block, hashKey)
|
|
239
|
+
padding.set(hash.subarray(0, 8)) // copy first 8 bytes of hash
|
|
240
|
+
hash.fill(0) // clear nonce buffer
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function encryptBlock(index, block, id, blockKey, hashKey) {
|
|
244
|
+
const padding = block.subarray(0, AutobeeEncryption.PADDING)
|
|
245
|
+
block = block.subarray(AutobeeEncryption.PADDING)
|
|
246
|
+
|
|
247
|
+
blockhash(block, padding, hashKey)
|
|
248
|
+
c.uint32.encode({ start: 4, end: 8, buffer: padding }, id)
|
|
249
|
+
|
|
250
|
+
c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
|
|
251
|
+
|
|
252
|
+
padding[0] = 1 // version in plaintext
|
|
253
|
+
|
|
254
|
+
nonce.set(padding, 8, 16)
|
|
255
|
+
|
|
256
|
+
// The combination of index, key id, fork id and block hash is very likely
|
|
257
|
+
// to be unique for a given Hypercore and therefore our nonce is suitable
|
|
258
|
+
encrypt(block, nonce, blockKey)
|
|
259
|
+
}
|
package/lib/fast-forward.js
CHANGED
|
@@ -7,7 +7,7 @@ const migrations = require('./migrations.js')
|
|
|
7
7
|
const { LEGACY_AUTOBASE_VERSION, LEGACY_OPLOG_VERSION } = require('./constants.js')
|
|
8
8
|
|
|
9
9
|
const DEFAULT_OP_TIMEOUT = 5_000
|
|
10
|
-
const MIN_FF_GAP =
|
|
10
|
+
const MIN_FF_GAP = 16
|
|
11
11
|
|
|
12
12
|
class FastForward {
|
|
13
13
|
constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT, conservative = false } = {}) {
|
|
@@ -39,7 +39,7 @@ class FastForward {
|
|
|
39
39
|
|
|
40
40
|
static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
|
|
41
41
|
|
|
42
|
-
static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
|
|
42
|
+
static async fromHead(auto, head, trusted, { force = false, skipGap = false, timeout = 0 } = {}) {
|
|
43
43
|
const conservative = !force && auto._conservativeFF
|
|
44
44
|
|
|
45
45
|
const oplog = await FastForward.flushHead(auto, head, { timeout })
|
|
@@ -51,7 +51,7 @@ class FastForward {
|
|
|
51
51
|
// legacy nodes from non-indexers have no system info to fast-forward from
|
|
52
52
|
if (!oplog.op.views || !verified.op.views) return null
|
|
53
53
|
|
|
54
|
-
if (!force && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
|
|
54
|
+
if (!force && !skipGap && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
|
|
55
55
|
return null
|
|
56
56
|
}
|
|
57
57
|
|
|
@@ -61,7 +61,7 @@ class FastForward {
|
|
|
61
61
|
})
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
|
|
64
|
+
static async fromHeads(auto, heads, { force = false, skipGap = false, timeout = 0 } = {}) {
|
|
65
65
|
const reference = auto._workingView.view
|
|
66
66
|
|
|
67
67
|
const promises = []
|
|
@@ -71,14 +71,12 @@ class FastForward {
|
|
|
71
71
|
promises.push(FastForward.flushHead(auto, head, { timeout }))
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
const ops = await Promise.all(promises)
|
|
74
|
+
const ops = await FastForward.reindexedCandidates(auto, await Promise.all(promises), {
|
|
75
|
+
timeout: timeout || DEFAULT_OP_TIMEOUT
|
|
76
|
+
})
|
|
75
77
|
if (auto.fastForwarding || auto.fastForwardTo) return null
|
|
76
78
|
|
|
77
|
-
const trust = await Promise.all(
|
|
78
|
-
ops.map((res) => {
|
|
79
|
-
return res === null ? false : auto.trusted.isTrusted(res.key, reference)
|
|
80
|
-
})
|
|
81
|
-
)
|
|
79
|
+
const trust = await Promise.all(ops.map((res) => auto.trusted.isTrusted(res.key, reference)))
|
|
82
80
|
if (auto.fastForwarding || auto.fastForwardTo) return null
|
|
83
81
|
|
|
84
82
|
const candidates = []
|
|
@@ -88,8 +86,7 @@ class FastForward {
|
|
|
88
86
|
|
|
89
87
|
for (let i = 0; i < ops.length; i++) {
|
|
90
88
|
const res = ops[i]
|
|
91
|
-
if (
|
|
92
|
-
if (!force && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
|
|
89
|
+
if (!force && !skipGap && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
|
|
93
90
|
|
|
94
91
|
if (!trust[i]) {
|
|
95
92
|
candidates.push(res)
|
|
@@ -103,7 +100,7 @@ class FastForward {
|
|
|
103
100
|
}
|
|
104
101
|
|
|
105
102
|
if (bestTrusted !== null) {
|
|
106
|
-
return FastForward.fromHead(auto, bestTrusted, null, { force, timeout })
|
|
103
|
+
return FastForward.fromHead(auto, bestTrusted, null, { force, skipGap, timeout })
|
|
107
104
|
}
|
|
108
105
|
|
|
109
106
|
candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
|
|
@@ -113,13 +110,38 @@ class FastForward {
|
|
|
113
110
|
if (trusted === null) continue
|
|
114
111
|
if (auto.fastForwarding || auto.fastForwardTo) return null
|
|
115
112
|
|
|
116
|
-
const ff = await FastForward.fromHead(auto, res, trusted, { force, timeout })
|
|
113
|
+
const ff = await FastForward.fromHead(auto, res, trusted, { force, skipGap, timeout })
|
|
117
114
|
if (ff !== null) return ff
|
|
118
115
|
}
|
|
119
116
|
|
|
120
117
|
return null
|
|
121
118
|
}
|
|
122
119
|
|
|
120
|
+
static async reindexedCandidates(auto, ops, { timeout }) {
|
|
121
|
+
const withViews = ops.filter((res) => res !== null && res.op.views)
|
|
122
|
+
const reindexed = await Promise.all(
|
|
123
|
+
withViews.map((res) => FastForward.isReindexed(auto, res, timeout))
|
|
124
|
+
)
|
|
125
|
+
return withViews.filter((res, i) => reindexed[i])
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
static async isReindexed(auto, res, timeout) {
|
|
129
|
+
const { system, view } = res.op.views
|
|
130
|
+
const checks = [system, view]
|
|
131
|
+
.filter((v) => !!v)
|
|
132
|
+
.map((range) => {
|
|
133
|
+
const { key, length } = batchToHead(range)
|
|
134
|
+
if (length === 0) return false
|
|
135
|
+
return auto._shouldReindex(key, { unknown: true, length, timeout })
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
for (const shouldReindex of await Promise.all(checks)) {
|
|
139
|
+
if (shouldReindex) return false
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
return true
|
|
143
|
+
}
|
|
144
|
+
|
|
123
145
|
static async flushHead(auto, head, opts) {
|
|
124
146
|
const core = auto.openCore(head.key)
|
|
125
147
|
|
package/lib/migrations.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
const c = require('compact-encoding')
|
|
2
2
|
const b4a = require('b4a')
|
|
3
3
|
const crypto = require('hypercore-crypto')
|
|
4
|
-
const { AutobeeEncryption } = require('autobee-encryption')
|
|
5
4
|
const { decodeBlock } = require('hyperbee2/lib/encoding.js')
|
|
6
5
|
|
|
6
|
+
const { AutobeeEncryption } = require('./encryption.js')
|
|
7
7
|
const encoding = require('./encoding.js')
|
|
8
8
|
const { assert, bail } = require('./asserts.js')
|
|
9
9
|
|
package/lib/writers.js
CHANGED
|
@@ -3,7 +3,7 @@ const encoding = require('./encoding.js')
|
|
|
3
3
|
const Triggers = require('./triggers.js')
|
|
4
4
|
const { LEGACY_OPLOG_VERSION } = require('./constants.js')
|
|
5
5
|
const { resolveWeight } = require('./witness.js')
|
|
6
|
-
const { WriterEncryption } = require('
|
|
6
|
+
const { WriterEncryption } = require('./encryption.js')
|
|
7
7
|
const ReadWriteLock = require('read-write-mutexify')
|
|
8
8
|
|
|
9
9
|
const WRITER_PREFETCH = 10
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "autobee",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.11.1",
|
|
4
4
|
"description": "Bee based autobase",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Holepunch Inc.",
|
|
@@ -23,12 +23,11 @@
|
|
|
23
23
|
"test:generate": "brittle -r test/all.js test/*.js"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
|
-
"autobee-encryption": "^0.1.2",
|
|
27
26
|
"autobee-wakeup": "^1.1.3",
|
|
28
27
|
"b4a": "^1.8.0",
|
|
29
|
-
"compact-encoding": "^3.
|
|
30
|
-
"hyperbee2": "^2.
|
|
31
|
-
"hypercore": "^11.
|
|
28
|
+
"compact-encoding": "^3.4.0",
|
|
29
|
+
"hyperbee2": "^2.17.1",
|
|
30
|
+
"hypercore": "^11.37.0",
|
|
32
31
|
"hypercore-id-encoding": "^1.3.0",
|
|
33
32
|
"hyperschema": "^1.26.2",
|
|
34
33
|
"read-write-mutexify": "^2.1.0",
|
|
@@ -36,6 +35,7 @@
|
|
|
36
35
|
"ready-resource": "^1.2.0",
|
|
37
36
|
"resolve-reject-promise": "^1.1.0",
|
|
38
37
|
"safety-catch": "^1.0.3",
|
|
38
|
+
"signal-promise": "^1.0.3",
|
|
39
39
|
"sodium-native": "^5.1.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|