autobee 2.10.1 → 2.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -5,15 +5,17 @@ const safetyCatch = require('safety-catch')
5
5
  const Hyperbee = require('hyperbee2')
6
6
  const ID = require('hypercore-id-encoding')
7
7
  const rrp = require('resolve-reject-promise')
8
- const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('autobee-encryption')
8
+ const Signal = require('signal-promise')
9
9
  const AutobeeWakeup = require('autobee-wakeup')
10
10
  const Hypercore = require('hypercore')
11
11
  const crypto = require('hypercore-crypto')
12
12
  const c = require('compact-encoding')
13
13
  const asserts = require('./lib/asserts.js')
14
14
  const boot = require('./lib/boot.js')
15
+ const { DEFAULT_MANIFEST_VERSION } = require('./lib/constants.js')
15
16
  const { resolveWeight, currentWeight } = require('./lib/witness.js')
16
17
  const encoding = require('./lib/encoding.js')
18
+ const { AutobeeEncryption, WriterEncryption, ViewEncryption } = require('./lib/encryption.js')
17
19
  const FastForward = require('./lib/fast-forward.js')
18
20
  const System = require('./lib/system.js')
19
21
  const ApplyCalls = require('./lib/apply-calls.js')
@@ -26,6 +28,7 @@ const migrations = require('./lib/migrations.js')
26
28
 
27
29
  const EMPTY_HEAD = { length: 0, key: null }
28
30
  const DEFAULT_ACK_THRESHOLD = 32
31
+ const BOOT_NETWORK_TIMEOUT = 5000
29
32
  const INTERRUPT = new Error('Apply interrupted')
30
33
 
31
34
  module.exports = class Autobee extends ReadyResource {
@@ -116,6 +119,9 @@ module.exports = class Autobee extends ReadyResource {
116
119
  this.fastForwardTo = null
117
120
  this._ffCandidates = new Map()
118
121
  this._ffSearching = null
122
+ this._bumpSignal = new Signal()
123
+ this._networkBooted = false
124
+ this._networkBooting = false
119
125
 
120
126
  this._workingBee = bee
121
127
  this._workingView = new ApplyView(this._workingBee, this)
@@ -143,6 +149,7 @@ module.exports = class Autobee extends ReadyResource {
143
149
  this._now = handlers.now || Date.now // overridable for clock-drift tests
144
150
  this._preapply = handlers.preapply || null
145
151
  this._preApplied = false
152
+ this._reindexed = false
146
153
  this._warmup = handlers.warmup || null
147
154
  this._hasApply = !!handlers.apply
148
155
  this._hasUpdate = !!handlers.update
@@ -205,7 +212,8 @@ module.exports = class Autobee extends ReadyResource {
205
212
  await 1
206
213
  const result = await this._prebooting
207
214
  return {
208
- name: 'autobee/' + result.local.id + '/' + name,
215
+ name: 'autobee/' + result.local.id + '/view/' + name,
216
+ manifestVersion: DEFAULT_MANIFEST_VERSION,
209
217
  encryption: name === 'system' ? this.getSystemEncryption() : this.getViewEncryption(),
210
218
  inflightRange: [256, 512]
211
219
  }
@@ -380,6 +388,7 @@ module.exports = class Autobee extends ReadyResource {
380
388
 
381
389
  async _close() {
382
390
  this._interrupting = true
391
+ this._bumpSignal.notify()
383
392
 
384
393
  // the local core holds the exclusive lock but the local writer closes it
385
394
  // early in the teardown, so hand the lock to a detached session that
@@ -583,6 +592,11 @@ module.exports = class Autobee extends ReadyResource {
583
592
  this._localFlushes = this.system.flushes
584
593
  }
585
594
 
595
+ async _isReindexing() {
596
+ if (this._networkBooted || !this._ffEnabled || this._migrating) return false
597
+ return !(await this._isReindexed())
598
+ }
599
+
586
600
  async _bootOnline() {
587
601
  if (!this._bootOnlineGuard.enter()) return
588
602
 
@@ -607,6 +621,7 @@ module.exports = class Autobee extends ReadyResource {
607
621
  }
608
622
 
609
623
  bumpSoon() {
624
+ this._bumpSignal.notify()
610
625
  this._bump(false).catch(safetyCatch)
611
626
  }
612
627
 
@@ -668,6 +683,61 @@ module.exports = class Autobee extends ReadyResource {
668
683
  this.emit('error', err)
669
684
  }
670
685
 
686
+ async compactMaybe() {
687
+ if (!this.writers.writable) return
688
+
689
+ if (await this._isReindexed()) return
690
+
691
+ await this._reindexBee(this._workingBee)
692
+ this.bee.move(this._workingBee.head())
693
+
694
+ await this._reindexBee(this.system.bee)
695
+
696
+ this._reindexed = true
697
+ }
698
+
699
+ async _isReindexed() {
700
+ const head = this.system.bee.head()
701
+ if (head === null) return true
702
+ if (await this._shouldReindex(head.key)) return false
703
+
704
+ const view = this.system.view
705
+ if (!view || !view.key) return true
706
+ return !(await this._shouldReindex(view.key))
707
+ }
708
+
709
+ async _reindexBee(bee) {
710
+ const head = bee.head()
711
+ const local = bee.context.local
712
+ if (head === null) return
713
+ if (await this._shouldReindex(local.key)) return
714
+ if (!(await this._shouldReindex(head.key))) return
715
+
716
+ // a non-empty local core is a completed reindex whose head was never stored
717
+ if (local.length === 0) {
718
+ await bee.reindex(async (change) => !(await this._shouldReindex(change.head.key)))
719
+ }
720
+
721
+ bee.move({ key: local.key, length: local.length })
722
+ }
723
+
724
+ async _shouldReindex(key, { unknown = false, length = 0, timeout = 0 } = {}) {
725
+ const core = this.store.get({ key, active: false })
726
+
727
+ try {
728
+ await core.ready()
729
+
730
+ if (core.manifest === null && length > 0) {
731
+ await core.get(length - 1, { raw: true, timeout }).catch(safetyCatch)
732
+ }
733
+
734
+ if (core.manifest === null) return unknown
735
+ return core.manifest.version > 1 && core.manifest.version < DEFAULT_MANIFEST_VERSION
736
+ } finally {
737
+ await core.close()
738
+ }
739
+ }
740
+
671
741
  // one-shot user gate: nothing applies until the host has resolved whatever
672
742
  // state apply depends on (e.g. legacy views recorded by a migration)
673
743
  async _runPreApply() {
@@ -688,6 +758,7 @@ module.exports = class Autobee extends ReadyResource {
688
758
  const { head = null, legacy = null } = this.bootFrom
689
759
 
690
760
  this.bootFrom = null
761
+ this._networkBooted = true
691
762
 
692
763
  if (legacy) {
693
764
  await this._bootFromSystem(legacy)
@@ -695,8 +766,14 @@ module.exports = class Autobee extends ReadyResource {
695
766
  this._wakeup.hint({ key: head.key, length: head.length || 0 })
696
767
  await this._bootFromHead(head)
697
768
  }
769
+ } else if (await this._isReindexing()) {
770
+ this._networkBooted = true
771
+ await this._bootFromNetwork()
698
772
  }
699
773
 
774
+ // preferably get a peer's compacted view during bootFrom
775
+ if (this.fastForwardTo === null) await this.compactMaybe()
776
+
700
777
  const changes = this._hasUpdate ? new UpdateChanges(this) : null
701
778
  if (changes) changes.track()
702
779
 
@@ -865,6 +942,7 @@ module.exports = class Autobee extends ReadyResource {
865
942
  const heads = await this._readCandidateHeads(hints, FastForward.DEFAULT_TIMEOUT)
866
943
 
867
944
  const ff = await FastForward.fromHeads(this, heads, {
945
+ skipGap: this._networkBooting,
868
946
  timeout: FastForward.DEFAULT_TIMEOUT
869
947
  })
870
948
 
@@ -1208,12 +1286,15 @@ module.exports = class Autobee extends ReadyResource {
1208
1286
  }
1209
1287
 
1210
1288
  async _appendAck() {
1211
- if (!this._acking) return false
1289
+ if (!this._acking && !this._reindexed) return false
1212
1290
  if (!this.writers.writable) return false
1213
1291
 
1214
1292
  if (this.writers.localWriter.pending !== null) return false
1215
- if ((await this._flushesBehind()) < this._ackThreshold) return false
1216
- if (await this._allHeadsTrusted()) return false
1293
+
1294
+ if (!this._reindexed) {
1295
+ if ((await this._flushesBehind()) < this._ackThreshold) return false
1296
+ if (await this._allHeadsTrusted()) return false
1297
+ }
1217
1298
 
1218
1299
  const links = this.system.getLinks(this.local.key)
1219
1300
  const now = this._now()
@@ -1371,6 +1452,14 @@ module.exports = class Autobee extends ReadyResource {
1371
1452
 
1372
1453
  const changed = await this.system.flush(batch, this._workingBee)
1373
1454
 
1455
+ if (this._reindexed && (await this._isReindexed())) {
1456
+ this._reindexed = false
1457
+ await this.local.setUserData(
1458
+ 'autobee/head',
1459
+ encoding.encodeBootRecord(this.system.bootRecord())
1460
+ )
1461
+ }
1462
+
1374
1463
  if (this.system.promotions.changed) this._prefetchApprovals().catch(safetyCatch)
1375
1464
 
1376
1465
  if (local) {
@@ -1551,6 +1640,41 @@ module.exports = class Autobee extends ReadyResource {
1551
1640
  }
1552
1641
  }
1553
1642
 
1643
+ async _bootFromNetwork() {
1644
+ this._networkBooting = true
1645
+
1646
+ try {
1647
+ await this._waitForNetworkBoot()
1648
+ } finally {
1649
+ this._networkBooting = false
1650
+ }
1651
+ }
1652
+
1653
+ async _waitForNetworkBoot() {
1654
+ const deadline = Date.now() + BOOT_NETWORK_TIMEOUT
1655
+
1656
+ this._requestWakeup()
1657
+
1658
+ while (!this._interrupting && !this.closing && this.fastForwardTo === null) {
1659
+ const hints = await this._applyWakeupHints()
1660
+ if (hints.length) this._queueFastForward(hints)
1661
+
1662
+ const remaining = deadline - Date.now()
1663
+ if (remaining <= 0 || this.fastForwardTo !== null) return
1664
+
1665
+ if (this._ffSearching !== null) {
1666
+ await Promise.race([this._ffSearching, this._bumpSignal.wait(remaining)])
1667
+ this._bumpSignal.notify()
1668
+ if (this._ffSearching === null) return
1669
+ continue
1670
+ }
1671
+
1672
+ if (hints.length) return
1673
+
1674
+ await this._bumpSignal.wait(remaining)
1675
+ }
1676
+ }
1677
+
1554
1678
  async _runFastForward(ff, { force = false } = {}) {
1555
1679
  if (this.fastForwardTo !== null || this.fastForwarding !== null) {
1556
1680
  await ff.close()
package/lib/constants.js CHANGED
@@ -1,10 +1,12 @@
1
1
  const LEGACY_OPLOG_VERSION = 2
2
2
  const OPLOG_VERSION = 4
3
+ const DEFAULT_MANIFEST_VERSION = 3
3
4
  const LEGACY_AUTOBASE_VERSION = 2
4
5
  const AUTOBEE_VERSION = 3
5
6
  const DEFAULT_OP_TIMEOUT = 5000
6
7
 
7
8
  module.exports = {
9
+ DEFAULT_MANIFEST_VERSION,
8
10
  LEGACY_OPLOG_VERSION,
9
11
  OPLOG_VERSION,
10
12
  LEGACY_AUTOBASE_VERSION,
package/lib/encoding.js CHANGED
@@ -1,10 +1,10 @@
1
1
  const b4a = require('b4a')
2
2
  const c = require('compact-encoding')
3
3
  const crypto = require('hypercore-crypto')
4
- const { AutobeeEncryption, WriterEncryption } = require('autobee-encryption')
5
4
 
6
5
  const { getEncoding } = require('../encoding/spec/autobee')
7
- const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants')
6
+ const { LEGACY_OPLOG_VERSION, OPLOG_VERSION } = require('./constants.js')
7
+ const { AutobeeEncryption, WriterEncryption } = require('./encryption.js')
8
8
 
9
9
  const Oplog = getEncoding('@autobee/oplog')
10
10
  const Link = getEncoding('@autobee/link')
@@ -0,0 +1,259 @@
1
+ const HypercoreEncryption = require('hypercore/lib/default-encryption.js')
2
+ const sodium = require('sodium-native')
3
+ const crypto = require('hypercore-crypto')
4
+ const c = require('compact-encoding')
5
+ const b4a = require('b4a')
6
+
7
+ const { DEFAULT_MANIFEST_VERSION } = require('./constants.js')
8
+
9
+ const ManifestData = {
10
+ preencode(state, m) {
11
+ c.uint.preencode(state, m.version)
12
+ state.end++ // max flag is 2 so always one byte
13
+
14
+ if (m.legacyBlocks) c.uint.preencode(state, m.legacyBlocks)
15
+ if (m.namespace) c.fixed32.preencode(state, m.namespace)
16
+ },
17
+ encode(state, m) {
18
+ const flags = (m.legacyBlocks ? 1 : 0) | (m.namespace ? 2 : 0)
19
+
20
+ c.uint.encode(state, m.version)
21
+ c.uint.encode(state, flags)
22
+
23
+ if (m.legacyBlocks) c.uint.encode(state, m.legacyBlocks)
24
+ if (m.namespace) c.fixed32.encode(state, m.namespace)
25
+ },
26
+ decode(state) {
27
+ const r0 = c.uint.decode(state)
28
+ const flags = c.uint.decode(state)
29
+
30
+ return {
31
+ version: r0,
32
+ legacyBlocks: (flags & 1) !== 0 ? c.uint.decode(state) : 0,
33
+ namespace: (flags & 2) !== 0 ? c.fixed32.decode(state) : null
34
+ }
35
+ }
36
+ }
37
+
38
+ const [, NS_VIEW_BLOCK_KEY, NS_HASH_KEY] = crypto.namespace('autobase', 4)
39
+
40
+ const [GENESIS_ENTROPY] = crypto.namespace('autobase/entropy', 2)
41
+
42
+ const nonce = b4a.alloc(sodium.crypto_stream_NONCEBYTES)
43
+ const hash = nonce.subarray(0, sodium.crypto_generichash_BYTES_MIN)
44
+
45
+ class AutobeeEncryption {
46
+ static PADDING = 8
47
+
48
+ constructor(auto) {
49
+ this.auto = auto
50
+ }
51
+
52
+ padding() {
53
+ return AutobeeEncryption.PADDING
54
+ }
55
+
56
+ isCompat(ctx) {
57
+ return ctx.manifest.version <= 1
58
+ }
59
+
60
+ async getKeys(id, ctx) {
61
+ if (!this.auto.encryptionKey) return null
62
+
63
+ const entropy = GENESIS_ENTROPY
64
+ const block = this.blockKey(entropy, ctx)
65
+ const hash = crypto.hash([NS_HASH_KEY, block])
66
+
67
+ return {
68
+ id,
69
+ block,
70
+ hash
71
+ }
72
+ }
73
+
74
+ blockKey(entropy, ctx) {
75
+ return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
76
+ }
77
+
78
+ compatKeys() {
79
+ throw new Error('Compatibility method is not specified')
80
+ }
81
+
82
+ async encrypt(index, block, fork, ctx) {
83
+ // writer cores are long lasting, but view cores should be up to date
84
+ if (this instanceof ViewEncryption && ctx.manifest.version < DEFAULT_MANIFEST_VERSION) {
85
+ throw new Error(`Unexpected manifest: version < ${DEFAULT_MANIFEST_VERSION}`)
86
+ }
87
+
88
+ if (this.isCompat(ctx, index)) {
89
+ const compat = this.compatKeys(ctx)
90
+ return HypercoreEncryption.encrypt(index, block, fork, compat.block, compat.blinding)
91
+ }
92
+
93
+ const keys = await this.getKeys(0, ctx)
94
+
95
+ encryptBlock(index, block, keys.id, keys.block, keys.hash)
96
+ }
97
+
98
+ async decrypt(index, block, ctx) {
99
+ if (this.isCompat(ctx, index)) {
100
+ return HypercoreEncryption.decrypt(index, block, this.compatKeys(ctx).block)
101
+ }
102
+
103
+ const padding = block.subarray(0, AutobeeEncryption.PADDING)
104
+ block = block.subarray(AutobeeEncryption.PADDING)
105
+
106
+ const type = padding[0]
107
+ switch (type) {
108
+ case 0:
109
+ return block // unencrypted
110
+
111
+ case 1:
112
+ break
113
+
114
+ default:
115
+ throw new Error('Unrecognised encryption type')
116
+ }
117
+
118
+ const id = c.uint32.decode({ start: 4, end: 8, buffer: padding })
119
+
120
+ const keys = await this.getKeys(id, ctx)
121
+
122
+ c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
123
+ nonce.set(padding, 8, 16)
124
+
125
+ // Decrypt the block using the full nonce
126
+ decrypt(block, nonce, keys.block)
127
+ }
128
+
129
+ static encryptAnchor(block, bootstrap, encryptionKey, namespace) {
130
+ const entropy = GENESIS_ENTROPY
131
+
132
+ const blockKey = getBlockKey(bootstrap, encryptionKey, entropy, namespace)
133
+ const hashKey = crypto.hash([NS_HASH_KEY, block])
134
+
135
+ encryptBlock(0, block, 0, blockKey, hashKey)
136
+ }
137
+
138
+ static async setSystemEncryption(bootstrap, encryptionKey, core, opts) {
139
+ await core.ready()
140
+ if (core.manifest.version === 1) {
141
+ const key = getCompatBlockKey(bootstrap, encryptionKey, '_system')
142
+ core.setEncryptionKey(key, { block: true })
143
+ return
144
+ }
145
+
146
+ const info = {
147
+ key: bootstrap,
148
+ encryptionKey
149
+ }
150
+
151
+ return core.setEncryption(new ViewEncryption(info, '_system'))
152
+ }
153
+
154
+ static getSystemEncryption(bootstrap, encryptionKey) {
155
+ return AutobeeEncryption.getViewEncryption(bootstrap, encryptionKey, '_system')
156
+ }
157
+
158
+ static getViewEncryption(bootstrap, encryptionKey, name) {
159
+ const info = { key: bootstrap, encryptionKey }
160
+ return new ViewEncryption(info, name)
161
+ }
162
+
163
+ static getWriterEncryption(bootstrap, encryptionKey) {
164
+ return new WriterEncryption({ key: bootstrap, encryptionKey })
165
+ }
166
+ }
167
+
168
+ class ViewEncryption extends AutobeeEncryption {
169
+ constructor(auto, name) {
170
+ super(auto)
171
+ this.name = name
172
+ }
173
+
174
+ compatKeys() {
175
+ const block = getCompatBlockKey(this.auto.key, this.auto.encryptionKey, this.name)
176
+ return {
177
+ block,
178
+ blinding: crypto.hash(block)
179
+ }
180
+ }
181
+
182
+ blockKey(entropy, ctx) {
183
+ // compat encryption
184
+ if (ctx.manifest.version < DEFAULT_MANIFEST_VERSION) {
185
+ return getCompatBlockKey(this.auto.key, entropy, this.name)
186
+ }
187
+
188
+ return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
189
+ }
190
+ }
191
+
192
+ class WriterEncryption extends AutobeeEncryption {
193
+ compatKeys(ctx) {
194
+ return HypercoreEncryption.deriveKeys(this.auto.encryptionKey, ctx.key)
195
+ }
196
+
197
+ blockKey(entropy, ctx) {
198
+ // used for anchors
199
+ if (ctx.manifest.userData) {
200
+ const userData = c.decode(ManifestData, ctx.manifest.userData)
201
+ if (userData.namespace !== null) {
202
+ const b = getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, userData.namespace)
203
+ return b
204
+ }
205
+ }
206
+
207
+ return getBlockKey(this.auto.key, this.auto.encryptionKey, entropy, ctx.key)
208
+ }
209
+ }
210
+
211
+ module.exports = {
212
+ AutobeeEncryption,
213
+ WriterEncryption,
214
+ ViewEncryption
215
+ }
216
+
217
+ function encrypt(block, nonce, key) {
218
+ sodium.crypto_stream_xor(block, block, nonce, key)
219
+ }
220
+
221
+ function decrypt(block, nonce, key) {
222
+ return encrypt(block, nonce, key) // symmetric
223
+ }
224
+
225
+ function getBlockKey(bootstrap, encryptionKey, entropy, hypercoreKey) {
226
+ return (
227
+ encryptionKey &&
228
+ crypto.hash([NS_VIEW_BLOCK_KEY, bootstrap, encryptionKey, entropy, hypercoreKey])
229
+ )
230
+ }
231
+
232
+ function getCompatBlockKey(bootstrap, encryptionKey, name) {
233
+ if (typeof name === 'string') return getCompatBlockKey(bootstrap, encryptionKey, b4a.from(name))
234
+ return encryptionKey && crypto.hash([NS_VIEW_BLOCK_KEY, bootstrap, encryptionKey, name])
235
+ }
236
+
237
+ function blockhash(block, padding, hashKey) {
238
+ sodium.crypto_generichash(hash, block, hashKey)
239
+ padding.set(hash.subarray(0, 8)) // copy first 8 bytes of hash
240
+ hash.fill(0) // clear nonce buffer
241
+ }
242
+
243
+ function encryptBlock(index, block, id, blockKey, hashKey) {
244
+ const padding = block.subarray(0, AutobeeEncryption.PADDING)
245
+ block = block.subarray(AutobeeEncryption.PADDING)
246
+
247
+ blockhash(block, padding, hashKey)
248
+ c.uint32.encode({ start: 4, end: 8, buffer: padding }, id)
249
+
250
+ c.uint64.encode({ start: 0, end: 8, buffer: nonce }, index)
251
+
252
+ padding[0] = 1 // version in plaintext
253
+
254
+ nonce.set(padding, 8, 16)
255
+
256
+ // The combination of index, key id, fork id and block hash is very likely
257
+ // to be unique for a given Hypercore and therefore our nonce is suitable
258
+ encrypt(block, nonce, blockKey)
259
+ }
@@ -7,7 +7,7 @@ const migrations = require('./migrations.js')
7
7
  const { LEGACY_AUTOBASE_VERSION, LEGACY_OPLOG_VERSION } = require('./constants.js')
8
8
 
9
9
  const DEFAULT_OP_TIMEOUT = 5_000
10
- const MIN_FF_GAP = 32
10
+ const MIN_FF_GAP = 16
11
11
 
12
12
  class FastForward {
13
13
  constructor(auto, head, { timeout = DEFAULT_OP_TIMEOUT, conservative = false } = {}) {
@@ -39,7 +39,7 @@ class FastForward {
39
39
 
40
40
  static DEFAULT_TIMEOUT = DEFAULT_OP_TIMEOUT
41
41
 
42
- static async fromHead(auto, head, trusted, { force = false, timeout = 0 } = {}) {
42
+ static async fromHead(auto, head, trusted, { force = false, skipGap = false, timeout = 0 } = {}) {
43
43
  const conservative = !force && auto._conservativeFF
44
44
 
45
45
  const oplog = await FastForward.flushHead(auto, head, { timeout })
@@ -51,7 +51,7 @@ class FastForward {
51
51
  // legacy nodes from non-indexers have no system info to fast-forward from
52
52
  if (!oplog.op.views || !verified.op.views) return null
53
53
 
54
- if (!force && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
54
+ if (!force && !skipGap && verified.op.views.flushes - auto.system.flushes < MIN_FF_GAP) {
55
55
  return null
56
56
  }
57
57
 
@@ -61,7 +61,7 @@ class FastForward {
61
61
  })
62
62
  }
63
63
 
64
- static async fromHeads(auto, heads, { force = false, timeout = 0 } = {}) {
64
+ static async fromHeads(auto, heads, { force = false, skipGap = false, timeout = 0 } = {}) {
65
65
  const reference = auto._workingView.view
66
66
 
67
67
  const promises = []
@@ -71,14 +71,12 @@ class FastForward {
71
71
  promises.push(FastForward.flushHead(auto, head, { timeout }))
72
72
  }
73
73
 
74
- const ops = await Promise.all(promises)
74
+ const ops = await FastForward.reindexedCandidates(auto, await Promise.all(promises), {
75
+ timeout: timeout || DEFAULT_OP_TIMEOUT
76
+ })
75
77
  if (auto.fastForwarding || auto.fastForwardTo) return null
76
78
 
77
- const trust = await Promise.all(
78
- ops.map((res) => {
79
- return res === null ? false : auto.trusted.isTrusted(res.key, reference)
80
- })
81
- )
79
+ const trust = await Promise.all(ops.map((res) => auto.trusted.isTrusted(res.key, reference)))
82
80
  if (auto.fastForwarding || auto.fastForwardTo) return null
83
81
 
84
82
  const candidates = []
@@ -88,8 +86,7 @@ class FastForward {
88
86
 
89
87
  for (let i = 0; i < ops.length; i++) {
90
88
  const res = ops[i]
91
- if (res === null || !res.op.views) continue
92
- if (!force && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
89
+ if (!force && !skipGap && res.op.views.flushes - auto.system.flushes < MIN_FF_GAP) continue
93
90
 
94
91
  if (!trust[i]) {
95
92
  candidates.push(res)
@@ -103,7 +100,7 @@ class FastForward {
103
100
  }
104
101
 
105
102
  if (bestTrusted !== null) {
106
- return FastForward.fromHead(auto, bestTrusted, null, { force, timeout })
103
+ return FastForward.fromHead(auto, bestTrusted, null, { force, skipGap, timeout })
107
104
  }
108
105
 
109
106
  candidates.sort((a, b) => b.op.views.flushes - a.op.views.flushes)
@@ -113,13 +110,38 @@ class FastForward {
113
110
  if (trusted === null) continue
114
111
  if (auto.fastForwarding || auto.fastForwardTo) return null
115
112
 
116
- const ff = await FastForward.fromHead(auto, res, trusted, { force, timeout })
113
+ const ff = await FastForward.fromHead(auto, res, trusted, { force, skipGap, timeout })
117
114
  if (ff !== null) return ff
118
115
  }
119
116
 
120
117
  return null
121
118
  }
122
119
 
120
+ static async reindexedCandidates(auto, ops, { timeout }) {
121
+ const withViews = ops.filter((res) => res !== null && res.op.views)
122
+ const reindexed = await Promise.all(
123
+ withViews.map((res) => FastForward.isReindexed(auto, res, timeout))
124
+ )
125
+ return withViews.filter((res, i) => reindexed[i])
126
+ }
127
+
128
+ static async isReindexed(auto, res, timeout) {
129
+ const { system, view } = res.op.views
130
+ const checks = [system, view]
131
+ .filter((v) => !!v)
132
+ .map((range) => {
133
+ const { key, length } = batchToHead(range)
134
+ if (length === 0) return false
135
+ return auto._shouldReindex(key, { unknown: true, length, timeout })
136
+ })
137
+
138
+ for (const shouldReindex of await Promise.all(checks)) {
139
+ if (shouldReindex) return false
140
+ }
141
+
142
+ return true
143
+ }
144
+
123
145
  static async flushHead(auto, head, opts) {
124
146
  const core = auto.openCore(head.key)
125
147
 
package/lib/migrations.js CHANGED
@@ -1,9 +1,9 @@
1
1
  const c = require('compact-encoding')
2
2
  const b4a = require('b4a')
3
3
  const crypto = require('hypercore-crypto')
4
- const { AutobeeEncryption } = require('autobee-encryption')
5
4
  const { decodeBlock } = require('hyperbee2/lib/encoding.js')
6
5
 
6
+ const { AutobeeEncryption } = require('./encryption.js')
7
7
  const encoding = require('./encoding.js')
8
8
  const { assert, bail } = require('./asserts.js')
9
9
 
package/lib/writers.js CHANGED
@@ -3,7 +3,7 @@ const encoding = require('./encoding.js')
3
3
  const Triggers = require('./triggers.js')
4
4
  const { LEGACY_OPLOG_VERSION } = require('./constants.js')
5
5
  const { resolveWeight } = require('./witness.js')
6
- const { WriterEncryption } = require('autobee-encryption')
6
+ const { WriterEncryption } = require('./encryption.js')
7
7
  const ReadWriteLock = require('read-write-mutexify')
8
8
 
9
9
  const WRITER_PREFETCH = 10
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "autobee",
3
- "version": "2.10.1",
3
+ "version": "2.11.1",
4
4
  "description": "Bee based autobase",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Holepunch Inc.",
@@ -23,12 +23,11 @@
23
23
  "test:generate": "brittle -r test/all.js test/*.js"
24
24
  },
25
25
  "dependencies": {
26
- "autobee-encryption": "^0.1.2",
27
26
  "autobee-wakeup": "^1.1.3",
28
27
  "b4a": "^1.8.0",
29
- "compact-encoding": "^3.3.0",
30
- "hyperbee2": "^2.16.0",
31
- "hypercore": "^11.33.2",
28
+ "compact-encoding": "^3.4.0",
29
+ "hyperbee2": "^2.17.1",
30
+ "hypercore": "^11.37.0",
32
31
  "hypercore-id-encoding": "^1.3.0",
33
32
  "hyperschema": "^1.26.2",
34
33
  "read-write-mutexify": "^2.1.0",
@@ -36,6 +35,7 @@
36
35
  "ready-resource": "^1.2.0",
37
36
  "resolve-reject-promise": "^1.1.0",
38
37
  "safety-catch": "^1.0.3",
38
+ "signal-promise": "^1.0.3",
39
39
  "sodium-native": "^5.1.0"
40
40
  },
41
41
  "devDependencies": {