@aztec/merkle-tree 0.16.1 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/index.d.ts +5 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +6 -2
- package/dest/interfaces/append_only_tree.d.ts +2 -1
- package/dest/interfaces/append_only_tree.d.ts.map +1 -1
- package/dest/interfaces/indexed_tree.d.ts +38 -17
- package/dest/interfaces/indexed_tree.d.ts.map +1 -1
- package/dest/interfaces/merkle_tree.d.ts +7 -0
- package/dest/interfaces/merkle_tree.d.ts.map +1 -1
- package/dest/interfaces/update_only_tree.d.ts +3 -3
- package/dest/interfaces/update_only_tree.d.ts.map +1 -1
- package/dest/load_tree.d.ts +2 -1
- package/dest/load_tree.d.ts.map +1 -1
- package/dest/load_tree.js +1 -2
- package/dest/new_tree.d.ts +1 -1
- package/dest/new_tree.d.ts.map +1 -1
- package/dest/new_tree.js +2 -2
- package/dest/snapshots/append_only_snapshot.d.ts +30 -0
- package/dest/snapshots/append_only_snapshot.d.ts.map +1 -0
- package/dest/snapshots/append_only_snapshot.js +200 -0
- package/dest/snapshots/base_full_snapshot.d.ts +50 -0
- package/dest/snapshots/base_full_snapshot.d.ts.map +1 -0
- package/dest/snapshots/base_full_snapshot.js +179 -0
- package/dest/snapshots/full_snapshot.d.ts +22 -0
- package/dest/snapshots/full_snapshot.d.ts.map +1 -0
- package/dest/snapshots/full_snapshot.js +21 -0
- package/dest/snapshots/indexed_tree_snapshot.d.ts +15 -0
- package/dest/snapshots/indexed_tree_snapshot.d.ts.map +1 -0
- package/dest/snapshots/indexed_tree_snapshot.js +75 -0
- package/dest/snapshots/snapshot_builder.d.ts +76 -0
- package/dest/snapshots/snapshot_builder.d.ts.map +1 -0
- package/dest/snapshots/snapshot_builder.js +2 -0
- package/dest/snapshots/snapshot_builder_test_suite.d.ts +5 -0
- package/dest/snapshots/snapshot_builder_test_suite.d.ts.map +1 -0
- package/dest/snapshots/snapshot_builder_test_suite.js +163 -0
- package/dest/sparse_tree/sparse_tree.d.ts +5 -0
- package/dest/sparse_tree/sparse_tree.d.ts.map +1 -1
- package/dest/sparse_tree/sparse_tree.js +18 -1
- package/dest/standard_indexed_tree/standard_indexed_tree.d.ts +111 -81
- package/dest/standard_indexed_tree/standard_indexed_tree.d.ts.map +1 -1
- package/dest/standard_indexed_tree/standard_indexed_tree.js +225 -259
- package/dest/standard_indexed_tree/test/standard_indexed_tree_with_append.d.ts.map +1 -1
- package/dest/standard_indexed_tree/test/standard_indexed_tree_with_append.js +13 -19
- package/dest/standard_tree/standard_tree.d.ts +5 -0
- package/dest/standard_tree/standard_tree.d.ts.map +1 -1
- package/dest/standard_tree/standard_tree.js +24 -1
- package/dest/tree_base.d.ts +9 -4
- package/dest/tree_base.d.ts.map +1 -1
- package/dest/tree_base.js +16 -7
- package/package.json +4 -3
- package/src/index.ts +5 -1
- package/src/interfaces/append_only_tree.ts +2 -1
- package/src/interfaces/indexed_tree.ts +50 -28
- package/src/interfaces/merkle_tree.ts +8 -0
- package/src/interfaces/update_only_tree.ts +3 -4
- package/src/load_tree.ts +2 -2
- package/src/new_tree.ts +2 -2
- package/src/snapshots/append_only_snapshot.ts +243 -0
- package/src/snapshots/base_full_snapshot.ts +232 -0
- package/src/snapshots/full_snapshot.ts +26 -0
- package/src/snapshots/indexed_tree_snapshot.ts +108 -0
- package/src/snapshots/snapshot_builder.ts +84 -0
- package/src/snapshots/snapshot_builder_test_suite.ts +218 -0
- package/src/sparse_tree/sparse_tree.ts +16 -0
- package/src/standard_indexed_tree/standard_indexed_tree.ts +325 -304
- package/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +23 -21
- package/src/standard_tree/standard_tree.ts +21 -0
- package/src/tree_base.ts +28 -7
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { Hasher, SiblingPath } from '@aztec/types';
|
|
2
|
+
|
|
3
|
+
import { LevelUp } from 'levelup';
|
|
4
|
+
|
|
5
|
+
import { AppendOnlyTree } from '../interfaces/append_only_tree.js';
|
|
6
|
+
import { TreeBase } from '../tree_base.js';
|
|
7
|
+
import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js';
|
|
8
|
+
|
|
9
|
+
// stores the last block that modified this node
|
|
10
|
+
const nodeModifiedAtBlockKey = (treeName: string, level: number, index: bigint) =>
|
|
11
|
+
`snapshot:node:${treeName}:${level}:${index}:block`;
|
|
12
|
+
|
|
13
|
+
// stores the value of the node at the above block
|
|
14
|
+
const historicalNodeKey = (treeName: string, level: number, index: bigint) =>
|
|
15
|
+
`snapshot:node:${treeName}:${level}:${index}:value`;
|
|
16
|
+
|
|
17
|
+
// metadata for a snapshot
|
|
18
|
+
const snapshotRootKey = (treeName: string, block: number) => `snapshot:root:${treeName}:${block}`;
|
|
19
|
+
const snapshotNumLeavesKey = (treeName: string, block: number) => `snapshot:numLeaves:${treeName}:${block}`;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A more space-efficient way of storing snapshots of AppendOnlyTrees that trades space need for slower
|
|
23
|
+
* sibling path reads.
|
|
24
|
+
*
|
|
25
|
+
* Complexity:
|
|
26
|
+
*
|
|
27
|
+
* N - count of non-zero nodes in tree
|
|
28
|
+
* M - count of snapshots
|
|
29
|
+
* H - tree height
|
|
30
|
+
*
|
|
31
|
+
* Space complexity: O(N + M) (N nodes - stores the last snapshot for each node and M - ints, for each snapshot stores up to which leaf its written to)
|
|
32
|
+
* Sibling path access:
|
|
33
|
+
* Best case: O(H) database reads + O(1) hashes
|
|
34
|
+
* Worst case: O(H) database reads + O(H) hashes
|
|
35
|
+
*/
|
|
36
|
+
export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder {
|
|
37
|
+
constructor(private db: LevelUp, private tree: TreeBase & AppendOnlyTree, private hasher: Hasher) {}
|
|
38
|
+
async getSnapshot(block: number): Promise<TreeSnapshot> {
|
|
39
|
+
const meta = await this.#getSnapshotMeta(block);
|
|
40
|
+
|
|
41
|
+
if (typeof meta === 'undefined') {
|
|
42
|
+
throw new Error(`Snapshot for tree ${this.tree.getName()} at block ${block} does not exist`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return new AppendOnlySnapshot(this.db, block, meta.numLeaves, meta.root, this.tree, this.hasher);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async snapshot(block: number): Promise<TreeSnapshot> {
|
|
49
|
+
const meta = await this.#getSnapshotMeta(block);
|
|
50
|
+
if (typeof meta !== 'undefined') {
|
|
51
|
+
// no-op, we already have a snapshot
|
|
52
|
+
return new AppendOnlySnapshot(this.db, block, meta.numLeaves, meta.root, this.tree, this.hasher);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const batch = this.db.batch();
|
|
56
|
+
const root = this.tree.getRoot(false);
|
|
57
|
+
const depth = this.tree.getDepth();
|
|
58
|
+
const treeName = this.tree.getName();
|
|
59
|
+
const queue: [Buffer, number, bigint][] = [[root, 0, 0n]];
|
|
60
|
+
|
|
61
|
+
// walk the tree in BF and store latest nodes
|
|
62
|
+
while (queue.length > 0) {
|
|
63
|
+
const [node, level, index] = queue.shift()!;
|
|
64
|
+
|
|
65
|
+
const historicalValue = await this.db.get(historicalNodeKey(treeName, level, index)).catch(() => undefined);
|
|
66
|
+
if (!historicalValue || !node.equals(historicalValue)) {
|
|
67
|
+
// we've never seen this node before or it's different than before
|
|
68
|
+
// update the historical tree and tag it with the block that modified it
|
|
69
|
+
batch.put(nodeModifiedAtBlockKey(treeName, level, index), String(block));
|
|
70
|
+
batch.put(historicalNodeKey(treeName, level, index), node);
|
|
71
|
+
} else {
|
|
72
|
+
// if this node hasn't changed, that means, nothing below it has changed either
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (level + 1 > depth) {
|
|
77
|
+
// short circuit if we've reached the leaf level
|
|
78
|
+
// otherwise getNode might throw if we ask for the children of a leaf
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// these could be undefined because zero hashes aren't stored in the tree
|
|
83
|
+
const [lhs, rhs] = await Promise.all([
|
|
84
|
+
this.tree.getNode(level + 1, 2n * index),
|
|
85
|
+
this.tree.getNode(level + 1, 2n * index + 1n),
|
|
86
|
+
]);
|
|
87
|
+
|
|
88
|
+
if (lhs) {
|
|
89
|
+
queue.push([lhs, level + 1, 2n * index]);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (rhs) {
|
|
93
|
+
queue.push([rhs, level + 1, 2n * index + 1n]);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const numLeaves = this.tree.getNumLeaves(false);
|
|
98
|
+
batch.put(snapshotNumLeavesKey(treeName, block), String(numLeaves));
|
|
99
|
+
batch.put(snapshotRootKey(treeName, block), root);
|
|
100
|
+
await batch.write();
|
|
101
|
+
|
|
102
|
+
return new AppendOnlySnapshot(this.db, block, numLeaves, root, this.tree, this.hasher);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async #getSnapshotMeta(block: number): Promise<
|
|
106
|
+
| {
|
|
107
|
+
/** The root of the tree snapshot */
|
|
108
|
+
root: Buffer;
|
|
109
|
+
/** The number of leaves in the tree snapshot */
|
|
110
|
+
numLeaves: bigint;
|
|
111
|
+
}
|
|
112
|
+
| undefined
|
|
113
|
+
> {
|
|
114
|
+
try {
|
|
115
|
+
const treeName = this.tree.getName();
|
|
116
|
+
const root = await this.db.get(snapshotRootKey(treeName, block));
|
|
117
|
+
const numLeaves = BigInt(await this.db.get(snapshotNumLeavesKey(treeName, block)));
|
|
118
|
+
return { root, numLeaves };
|
|
119
|
+
} catch (err) {
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* a
|
|
127
|
+
*/
|
|
128
|
+
class AppendOnlySnapshot implements TreeSnapshot {
|
|
129
|
+
constructor(
|
|
130
|
+
private db: LevelUp,
|
|
131
|
+
private block: number,
|
|
132
|
+
private leafCount: bigint,
|
|
133
|
+
private historicalRoot: Buffer,
|
|
134
|
+
private tree: TreeBase & AppendOnlyTree,
|
|
135
|
+
private hasher: Hasher,
|
|
136
|
+
) {}
|
|
137
|
+
|
|
138
|
+
public async getSiblingPath<N extends number>(index: bigint): Promise<SiblingPath<N>> {
|
|
139
|
+
const path: Buffer[] = [];
|
|
140
|
+
const depth = this.tree.getDepth();
|
|
141
|
+
let level = depth;
|
|
142
|
+
|
|
143
|
+
while (level > 0) {
|
|
144
|
+
const isRight = index & 0x01n;
|
|
145
|
+
const siblingIndex = isRight ? index - 1n : index + 1n;
|
|
146
|
+
|
|
147
|
+
const sibling = await this.#getHistoricalNodeValue(level, siblingIndex);
|
|
148
|
+
path.push(sibling);
|
|
149
|
+
|
|
150
|
+
level -= 1;
|
|
151
|
+
index >>= 1n;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return new SiblingPath<N>(depth as N, path);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
getDepth(): number {
|
|
158
|
+
return this.tree.getDepth();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
getNumLeaves(): bigint {
|
|
162
|
+
return this.leafCount;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
getRoot(): Buffer {
|
|
166
|
+
// we could recompute it, but it's way cheaper to just store the root
|
|
167
|
+
return this.historicalRoot;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async getLeafValue(index: bigint): Promise<Buffer | undefined> {
|
|
171
|
+
const leafLevel = this.getDepth();
|
|
172
|
+
const blockNumber = await this.#getBlockNumberThatModifiedNode(leafLevel, index);
|
|
173
|
+
|
|
174
|
+
// leaf hasn't been set yet
|
|
175
|
+
if (typeof blockNumber === 'undefined') {
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// leaf was set some time in the past
|
|
180
|
+
if (blockNumber <= this.block) {
|
|
181
|
+
return this.db.get(historicalNodeKey(this.tree.getName(), leafLevel, index));
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// leaf has been set but in a block in the future
|
|
185
|
+
return undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async #getHistoricalNodeValue(level: number, index: bigint): Promise<Buffer> {
|
|
189
|
+
const blockNumber = await this.#getBlockNumberThatModifiedNode(level, index);
|
|
190
|
+
|
|
191
|
+
// node has never been set
|
|
192
|
+
if (typeof blockNumber === 'undefined') {
|
|
193
|
+
return this.tree.getZeroHash(level);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// node was set some time in the past
|
|
197
|
+
if (blockNumber <= this.block) {
|
|
198
|
+
return this.db.get(historicalNodeKey(this.tree.getName(), level, index));
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// the node has been modified since this snapshot was taken
|
|
202
|
+
// because we're working with an AppendOnly tree, historical leaves never change
|
|
203
|
+
// so what we do instead is rebuild this Merkle path up using zero hashes as needed
|
|
204
|
+
// worst case this will do O(H) hashes
|
|
205
|
+
//
|
|
206
|
+
// we first check if this subtree was touched by the block
|
|
207
|
+
// compare how many leaves this block added to the leaf interval of this subtree
|
|
208
|
+
// if they don't intersect then the whole subtree was a hash of zero
|
|
209
|
+
// if they do then we need to rebuild the merkle tree
|
|
210
|
+
const depth = this.tree.getDepth();
|
|
211
|
+
const leafStart = index * 2n ** BigInt(depth - level);
|
|
212
|
+
if (leafStart >= this.leafCount) {
|
|
213
|
+
return this.tree.getZeroHash(level);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const [lhs, rhs] = await Promise.all([
|
|
217
|
+
this.#getHistoricalNodeValue(level + 1, 2n * index),
|
|
218
|
+
this.#getHistoricalNodeValue(level + 1, 2n * index + 1n),
|
|
219
|
+
]);
|
|
220
|
+
|
|
221
|
+
return this.hasher.hash(lhs, rhs);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async #getBlockNumberThatModifiedNode(level: number, index: bigint): Promise<number | undefined> {
|
|
225
|
+
try {
|
|
226
|
+
const value: Buffer | string = await this.db.get(nodeModifiedAtBlockKey(this.tree.getName(), level, index));
|
|
227
|
+
return parseInt(value.toString(), 10);
|
|
228
|
+
} catch (err) {
|
|
229
|
+
return undefined;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async findLeafIndex(value: Buffer): Promise<bigint | undefined> {
|
|
234
|
+
const numLeaves = this.getNumLeaves();
|
|
235
|
+
for (let i = 0n; i < numLeaves; i++) {
|
|
236
|
+
const currentValue = await this.getLeafValue(i);
|
|
237
|
+
if (currentValue && currentValue.equals(value)) {
|
|
238
|
+
return i;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return undefined;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { SiblingPath } from '@aztec/types';
|
|
2
|
+
|
|
3
|
+
import { LevelUp, LevelUpChain } from 'levelup';
|
|
4
|
+
|
|
5
|
+
import { TreeBase } from '../tree_base.js';
|
|
6
|
+
import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js';
|
|
7
|
+
|
|
8
|
+
// key for a node's children
|
|
9
|
+
const snapshotChildKey = (node: Buffer, child: 0 | 1) =>
|
|
10
|
+
Buffer.concat([Buffer.from('snapshot:node:'), node, Buffer.from(':' + child)]);
|
|
11
|
+
|
|
12
|
+
// metadata for a snapshot
|
|
13
|
+
const snapshotRootKey = (treeName: string, block: number) => `snapshot:root:${treeName}:${block}`;
|
|
14
|
+
const snapshotNumLeavesKey = (treeName: string, block: number) => `snapshot:numLeaves:${treeName}:${block}`;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Builds a full snapshot of a tree. This implementation works for any Merkle tree and stores
|
|
18
|
+
* it in a database in a similar way to how a tree is stored in memory, using pointers.
|
|
19
|
+
*
|
|
20
|
+
* Sharing the same database between versions and trees is recommended as the trees would share
|
|
21
|
+
* structure.
|
|
22
|
+
*
|
|
23
|
+
* Implement the protected method `handleLeaf` to store any additional data you need for each leaf.
|
|
24
|
+
*
|
|
25
|
+
* Complexity:
|
|
26
|
+
* N - count of non-zero nodes in tree
|
|
27
|
+
* M - count of snapshots
|
|
28
|
+
* H - tree height
|
|
29
|
+
* Worst case space complexity: O(N * M)
|
|
30
|
+
* Sibling path access: O(H) database reads
|
|
31
|
+
*/
|
|
32
|
+
export abstract class BaseFullTreeSnapshotBuilder<T extends TreeBase, S extends TreeSnapshot>
|
|
33
|
+
implements TreeSnapshotBuilder<S>
|
|
34
|
+
{
|
|
35
|
+
constructor(protected db: LevelUp, protected tree: T) {}
|
|
36
|
+
|
|
37
|
+
async snapshot(block: number): Promise<S> {
|
|
38
|
+
const snapshotMetadata = await this.#getSnapshotMeta(block);
|
|
39
|
+
|
|
40
|
+
if (snapshotMetadata) {
|
|
41
|
+
return this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const batch = this.db.batch();
|
|
45
|
+
const root = this.tree.getRoot(false);
|
|
46
|
+
const numLeaves = this.tree.getNumLeaves(false);
|
|
47
|
+
const depth = this.tree.getDepth();
|
|
48
|
+
const queue: [Buffer, number, bigint][] = [[root, 0, 0n]];
|
|
49
|
+
|
|
50
|
+
// walk the tree breadth-first and store each of its nodes in the database
|
|
51
|
+
// for each node we save two keys
|
|
52
|
+
// <node hash>:0 -> <left child's hash>
|
|
53
|
+
// <node hash>:1 -> <right child's hash>
|
|
54
|
+
while (queue.length > 0) {
|
|
55
|
+
const [node, level, i] = queue.shift()!;
|
|
56
|
+
// check if the database already has a child for this tree
|
|
57
|
+
// if it does, then we know we've seen the whole subtree below it before
|
|
58
|
+
// and we don't have to traverse it anymore
|
|
59
|
+
// we use the left child here, but it could be anything that shows we've stored the node before
|
|
60
|
+
const exists: Buffer | undefined = await this.db.get(snapshotChildKey(node, 0)).catch(() => undefined);
|
|
61
|
+
if (exists) {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (level + 1 > depth) {
|
|
66
|
+
// short circuit if we've reached the leaf level
|
|
67
|
+
// otherwise getNode might throw if we ask for the children of a leaf
|
|
68
|
+
await this.handleLeaf(i, node, batch);
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const [lhs, rhs] = await Promise.all([
|
|
73
|
+
this.tree.getNode(level + 1, 2n * i),
|
|
74
|
+
this.tree.getNode(level + 1, 2n * i + 1n),
|
|
75
|
+
]);
|
|
76
|
+
|
|
77
|
+
// we want the zero hash at the children's level, not the node's level
|
|
78
|
+
const zeroHash = this.tree.getZeroHash(level + 1);
|
|
79
|
+
|
|
80
|
+
batch.put(snapshotChildKey(node, 0), lhs ?? zeroHash);
|
|
81
|
+
batch.put(snapshotChildKey(node, 1), rhs ?? zeroHash);
|
|
82
|
+
|
|
83
|
+
// enqueue the children only if they're not zero hashes
|
|
84
|
+
if (lhs) {
|
|
85
|
+
queue.push([lhs, level + 1, 2n * i]);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (rhs) {
|
|
89
|
+
queue.push([rhs, level + 1, 2n * i + 1n]);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
batch.put(snapshotRootKey(this.tree.getName(), block), root);
|
|
94
|
+
batch.put(snapshotNumLeavesKey(this.tree.getName(), block), String(numLeaves));
|
|
95
|
+
await batch.write();
|
|
96
|
+
|
|
97
|
+
return this.openSnapshot(root, numLeaves);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
protected handleLeaf(_index: bigint, _node: Buffer, _batch: LevelUpChain) {
|
|
101
|
+
return Promise.resolve();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async getSnapshot(version: number): Promise<S> {
|
|
105
|
+
const snapshotMetadata = await this.#getSnapshotMeta(version);
|
|
106
|
+
|
|
107
|
+
if (!snapshotMetadata) {
|
|
108
|
+
throw new Error(`Version ${version} does not exist for tree ${this.tree.getName()}`);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
protected abstract openSnapshot(root: Buffer, numLeaves: bigint): S;
|
|
115
|
+
|
|
116
|
+
async #getSnapshotMeta(block: number): Promise<
|
|
117
|
+
| {
|
|
118
|
+
/** The root of the tree snapshot */
|
|
119
|
+
root: Buffer;
|
|
120
|
+
/** The number of leaves in the tree snapshot */
|
|
121
|
+
numLeaves: bigint;
|
|
122
|
+
}
|
|
123
|
+
| undefined
|
|
124
|
+
> {
|
|
125
|
+
try {
|
|
126
|
+
const treeName = this.tree.getName();
|
|
127
|
+
const root = await this.db.get(snapshotRootKey(treeName, block));
|
|
128
|
+
const numLeaves = BigInt(await this.db.get(snapshotNumLeavesKey(treeName, block)));
|
|
129
|
+
return { root, numLeaves };
|
|
130
|
+
} catch (err) {
|
|
131
|
+
return undefined;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* A source of sibling paths from a snapshot tree
|
|
138
|
+
*/
|
|
139
|
+
export class BaseFullTreeSnapshot implements TreeSnapshot {
|
|
140
|
+
constructor(
|
|
141
|
+
protected db: LevelUp,
|
|
142
|
+
protected historicRoot: Buffer,
|
|
143
|
+
protected numLeaves: bigint,
|
|
144
|
+
protected tree: TreeBase,
|
|
145
|
+
) {}
|
|
146
|
+
|
|
147
|
+
async getSiblingPath<N extends number>(index: bigint): Promise<SiblingPath<N>> {
|
|
148
|
+
const siblings: Buffer[] = [];
|
|
149
|
+
|
|
150
|
+
for await (const [_node, sibling] of this.pathFromRootToLeaf(index)) {
|
|
151
|
+
siblings.push(sibling);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// we got the siblings we were looking for, but they are in root-leaf order
|
|
155
|
+
// reverse them here so we have leaf-root (what SiblingPath expects)
|
|
156
|
+
siblings.reverse();
|
|
157
|
+
|
|
158
|
+
return new SiblingPath<N>(this.tree.getDepth() as N, siblings);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async getLeafValue(index: bigint): Promise<Buffer | undefined> {
|
|
162
|
+
let leafNode: Buffer | undefined = undefined;
|
|
163
|
+
for await (const [node, _sibling] of this.pathFromRootToLeaf(index)) {
|
|
164
|
+
leafNode = node;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return leafNode;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
getDepth(): number {
|
|
171
|
+
return this.tree.getDepth();
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
getRoot(): Buffer {
|
|
175
|
+
return this.historicRoot;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
getNumLeaves(): bigint {
|
|
179
|
+
return this.numLeaves;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
protected async *pathFromRootToLeaf(leafIndex: bigint) {
|
|
183
|
+
const root = this.historicRoot;
|
|
184
|
+
const pathFromRoot = this.#getPathFromRoot(leafIndex);
|
|
185
|
+
|
|
186
|
+
let node: Buffer = root;
|
|
187
|
+
for (let i = 0; i < pathFromRoot.length; i++) {
|
|
188
|
+
// get both children. We'll need both anyway (one to keep track of, the other to walk down to)
|
|
189
|
+
const children: [Buffer, Buffer] = await Promise.all([
|
|
190
|
+
this.db.get(snapshotChildKey(node, 0)),
|
|
191
|
+
this.db.get(snapshotChildKey(node, 1)),
|
|
192
|
+
]).catch(() => [this.tree.getZeroHash(i + 1), this.tree.getZeroHash(i + 1)]);
|
|
193
|
+
const next = children[pathFromRoot[i]];
|
|
194
|
+
const sibling = children[(pathFromRoot[i] + 1) % 2];
|
|
195
|
+
|
|
196
|
+
yield [next, sibling];
|
|
197
|
+
|
|
198
|
+
node = next;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Calculates the path from the root to the target leaf. Returns an array of 0s and 1s,
|
|
204
|
+
* each 0 represents walking down a left child and each 1 walking down to the child on the right.
|
|
205
|
+
*
|
|
206
|
+
* @param leafIndex - The target leaf
|
|
207
|
+
* @returns An array of 0s and 1s
|
|
208
|
+
*/
|
|
209
|
+
#getPathFromRoot(leafIndex: bigint): ReadonlyArray<0 | 1> {
|
|
210
|
+
const path: Array<0 | 1> = [];
|
|
211
|
+
let level = this.tree.getDepth();
|
|
212
|
+
while (level > 0) {
|
|
213
|
+
path.push(leafIndex & 0x01n ? 1 : 0);
|
|
214
|
+
leafIndex >>= 1n;
|
|
215
|
+
level--;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
path.reverse();
|
|
219
|
+
return path;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async findLeafIndex(value: Buffer): Promise<bigint | undefined> {
|
|
223
|
+
const numLeaves = this.getNumLeaves();
|
|
224
|
+
for (let i = 0n; i < numLeaves; i++) {
|
|
225
|
+
const currentValue = await this.getLeafValue(i);
|
|
226
|
+
if (currentValue && currentValue.equals(value)) {
|
|
227
|
+
return i;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return undefined;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { TreeBase } from '../tree_base.js';
|
|
2
|
+
import { BaseFullTreeSnapshot, BaseFullTreeSnapshotBuilder } from './base_full_snapshot.js';
|
|
3
|
+
import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Builds a full snapshot of a tree. This implementation works for any Merkle tree and stores
|
|
7
|
+
* it in a database in a similar way to how a tree is stored in memory, using pointers.
|
|
8
|
+
*
|
|
9
|
+
* Sharing the same database between versions and trees is recommended as the trees would share
|
|
10
|
+
* structure.
|
|
11
|
+
*
|
|
12
|
+
* Complexity:
|
|
13
|
+
* N - count of non-zero nodes in tree
|
|
14
|
+
* M - count of snapshots
|
|
15
|
+
* H - tree height
|
|
16
|
+
* Worst case space complexity: O(N * M)
|
|
17
|
+
* Sibling path access: O(H) database reads
|
|
18
|
+
*/
|
|
19
|
+
export class FullTreeSnapshotBuilder
|
|
20
|
+
extends BaseFullTreeSnapshotBuilder<TreeBase, TreeSnapshot>
|
|
21
|
+
implements TreeSnapshotBuilder<TreeSnapshot>
|
|
22
|
+
{
|
|
23
|
+
protected openSnapshot(root: Buffer, numLeaves: bigint): TreeSnapshot {
|
|
24
|
+
return new BaseFullTreeSnapshot(this.db, root, numLeaves, this.tree);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
|
|
2
|
+
|
|
3
|
+
import { LevelUp, LevelUpChain } from 'levelup';
|
|
4
|
+
|
|
5
|
+
import { IndexedTree } from '../interfaces/indexed_tree.js';
|
|
6
|
+
import { PreimageFactory } from '../standard_indexed_tree/standard_indexed_tree.js';
|
|
7
|
+
import { TreeBase } from '../tree_base.js';
|
|
8
|
+
import { BaseFullTreeSnapshot, BaseFullTreeSnapshotBuilder } from './base_full_snapshot.js';
|
|
9
|
+
import { IndexedTreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js';
|
|
10
|
+
|
|
11
|
+
const snapshotLeafValue = (node: Buffer, index: bigint) =>
|
|
12
|
+
Buffer.concat([Buffer.from('snapshot:leaf:'), node, Buffer.from(':' + index)]);
|
|
13
|
+
|
|
14
|
+
/** a */
|
|
15
|
+
export class IndexedTreeSnapshotBuilder
|
|
16
|
+
extends BaseFullTreeSnapshotBuilder<IndexedTree & TreeBase, IndexedTreeSnapshot>
|
|
17
|
+
implements TreeSnapshotBuilder<IndexedTreeSnapshot>
|
|
18
|
+
{
|
|
19
|
+
constructor(db: LevelUp, tree: IndexedTree & TreeBase, private leafPreimageBuilder: PreimageFactory) {
|
|
20
|
+
super(db, tree);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
protected openSnapshot(root: Buffer, numLeaves: bigint): IndexedTreeSnapshot {
|
|
24
|
+
return new IndexedTreeSnapshotImpl(this.db, root, numLeaves, this.tree, this.leafPreimageBuilder);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
protected async handleLeaf(index: bigint, node: Buffer, batch: LevelUpChain) {
|
|
28
|
+
const leafPreimage = await this.tree.getLatestLeafPreimageCopy(index, false);
|
|
29
|
+
if (leafPreimage) {
|
|
30
|
+
batch.put(snapshotLeafValue(node, index), leafPreimage.toBuffer());
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A snapshot of an indexed tree at a particular point in time */
|
|
36
|
+
class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTreeSnapshot {
|
|
37
|
+
constructor(
|
|
38
|
+
db: LevelUp,
|
|
39
|
+
historicRoot: Buffer,
|
|
40
|
+
numLeaves: bigint,
|
|
41
|
+
tree: IndexedTree & TreeBase,
|
|
42
|
+
private leafPreimageBuilder: PreimageFactory,
|
|
43
|
+
) {
|
|
44
|
+
super(db, historicRoot, numLeaves, tree);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async getLeafValue(index: bigint): Promise<Buffer | undefined> {
|
|
48
|
+
const leafPreimage = await this.getLatestLeafPreimageCopy(index);
|
|
49
|
+
return leafPreimage?.toBuffer();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async getLatestLeafPreimageCopy(index: bigint): Promise<IndexedTreeLeafPreimage | undefined> {
|
|
53
|
+
const leafNode = await super.getLeafValue(index);
|
|
54
|
+
const leafValue = await this.db.get(snapshotLeafValue(leafNode!, index)).catch(() => undefined);
|
|
55
|
+
if (leafValue) {
|
|
56
|
+
return this.leafPreimageBuilder.fromBuffer(leafValue);
|
|
57
|
+
} else {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async findIndexOfPreviousKey(newValue: bigint): Promise<{
|
|
63
|
+
/**
|
|
64
|
+
* The index of the found leaf.
|
|
65
|
+
*/
|
|
66
|
+
index: bigint;
|
|
67
|
+
/**
|
|
68
|
+
* A flag indicating if the corresponding leaf's value is equal to `newValue`.
|
|
69
|
+
*/
|
|
70
|
+
alreadyPresent: boolean;
|
|
71
|
+
}> {
|
|
72
|
+
const numLeaves = this.getNumLeaves();
|
|
73
|
+
const diff: bigint[] = [];
|
|
74
|
+
|
|
75
|
+
for (let i = 0; i < numLeaves; i++) {
|
|
76
|
+
// this is very inefficient
|
|
77
|
+
const storedLeaf = await this.getLatestLeafPreimageCopy(BigInt(i))!;
|
|
78
|
+
|
|
79
|
+
// The stored leaf can be undefined if it addresses an empty leaf
|
|
80
|
+
// If the leaf is empty we do the same as if the leaf was larger
|
|
81
|
+
if (storedLeaf === undefined) {
|
|
82
|
+
diff.push(newValue);
|
|
83
|
+
} else if (storedLeaf.getKey() > newValue) {
|
|
84
|
+
diff.push(newValue);
|
|
85
|
+
} else if (storedLeaf.getKey() === newValue) {
|
|
86
|
+
return { index: BigInt(i), alreadyPresent: true };
|
|
87
|
+
} else {
|
|
88
|
+
diff.push(newValue - storedLeaf.getKey());
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let minIndex = 0;
|
|
93
|
+
for (let i = 1; i < diff.length; i++) {
|
|
94
|
+
if (diff[i] < diff[minIndex]) {
|
|
95
|
+
minIndex = i;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { index: BigInt(minIndex), alreadyPresent: false };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async findLeafIndex(value: Buffer): Promise<bigint | undefined> {
|
|
103
|
+
const index = await this.tree.findLeafIndex(value, false);
|
|
104
|
+
if (index !== undefined && index < this.getNumLeaves()) {
|
|
105
|
+
return index;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
|
|
2
|
+
import { SiblingPath } from '@aztec/types';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* An interface for a tree that can record snapshots of its contents.
|
|
6
|
+
*/
|
|
7
|
+
export interface TreeSnapshotBuilder<S extends TreeSnapshot = TreeSnapshot> {
|
|
8
|
+
/**
|
|
9
|
+
* Creates a snapshot of the tree at the given version.
|
|
10
|
+
* @param block - The version to snapshot the tree at.
|
|
11
|
+
*/
|
|
12
|
+
snapshot(block: number): Promise<S>;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Returns a snapshot of the tree at the given version.
|
|
16
|
+
* @param block - The version of the snapshot to return.
|
|
17
|
+
*/
|
|
18
|
+
getSnapshot(block: number): Promise<S>;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A tree snapshot
|
|
23
|
+
*/
|
|
24
|
+
export interface TreeSnapshot {
|
|
25
|
+
/**
|
|
26
|
+
* Returns the current root of the tree.
|
|
27
|
+
*/
|
|
28
|
+
getRoot(): Buffer;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Returns the number of leaves in the tree.
|
|
32
|
+
*/
|
|
33
|
+
getDepth(): number;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Returns the number of leaves in the tree.
|
|
37
|
+
*/
|
|
38
|
+
getNumLeaves(): bigint;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Returns the value of a leaf at the specified index.
|
|
42
|
+
* @param index - The index of the leaf value to be returned.
|
|
43
|
+
*/
|
|
44
|
+
getLeafValue(index: bigint): Promise<Buffer | undefined>;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Returns the sibling path for a requested leaf index.
|
|
48
|
+
* @param index - The index of the leaf for which a sibling path is required.
|
|
49
|
+
*/
|
|
50
|
+
getSiblingPath<N extends number>(index: bigint): Promise<SiblingPath<N>>;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Returns the index of a leaf given its value, or undefined if no leaf with that value is found.
|
|
54
|
+
* @param treeId - The ID of the tree.
|
|
55
|
+
* @param value - The leaf value to look for.
|
|
56
|
+
* @returns The index of the first leaf found with a given value (undefined if not found).
|
|
57
|
+
*/
|
|
58
|
+
findLeafIndex(value: Buffer): Promise<bigint | undefined>;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** A snapshot of an indexed tree */
|
|
62
|
+
export interface IndexedTreeSnapshot extends TreeSnapshot {
|
|
63
|
+
/**
|
|
64
|
+
* Gets the historical data for a leaf
|
|
65
|
+
* @param index - The index of the leaf to get the data for
|
|
66
|
+
*/
|
|
67
|
+
getLatestLeafPreimageCopy(index: bigint): Promise<IndexedTreeLeafPreimage | undefined>;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Finds the index of the largest leaf whose value is less than or equal to the provided value.
|
|
71
|
+
* @param newValue - The new value to be inserted into the tree.
|
|
72
|
+
* @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`.
|
|
73
|
+
*/
|
|
74
|
+
findIndexOfPreviousKey(newValue: bigint): Promise<{
|
|
75
|
+
/**
|
|
76
|
+
* The index of the found leaf.
|
|
77
|
+
*/
|
|
78
|
+
index: bigint;
|
|
79
|
+
/**
|
|
80
|
+
* A flag indicating if the corresponding leaf's value is equal to `newValue`.
|
|
81
|
+
*/
|
|
82
|
+
alreadyPresent: boolean;
|
|
83
|
+
}>;
|
|
84
|
+
}
|