@aztec/kv-store 4.0.0-nightly.20250907 → 4.0.0-nightly.20260107
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +2 -2
- package/dest/config.js +2 -2
- package/dest/indexeddb/array.d.ts +1 -1
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +3 -1
- package/dest/indexeddb/index.d.ts +1 -1
- package/dest/indexeddb/index.js +1 -1
- package/dest/indexeddb/map.d.ts +1 -1
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/multi_map.d.ts +2 -1
- package/dest/indexeddb/multi_map.d.ts.map +1 -1
- package/dest/indexeddb/multi_map.js +14 -0
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +1 -1
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/store.d.ts +1 -1
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/interfaces/array.d.ts +1 -1
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/common.d.ts +1 -1
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +1 -1
- package/dest/interfaces/map.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/multi_map.d.ts +7 -1
- package/dest/interfaces/multi_map.d.ts.map +1 -1
- package/dest/interfaces/multi_map_test_suite.d.ts +1 -1
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/multi_map_test_suite.js +94 -0
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/store.d.ts +1 -1
- package/dest/interfaces/utils.d.ts +1 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/lmdb/array.d.ts +1 -1
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/array.js +4 -2
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +1 -1
- package/dest/lmdb/index.js +2 -2
- package/dest/lmdb/map.d.ts +1 -1
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/multi_map.d.ts +2 -1
- package/dest/lmdb/multi_map.d.ts.map +1 -1
- package/dest/lmdb/multi_map.js +15 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +2 -3
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +1 -1
- package/dest/lmdb-v2/array.d.ts +1 -1
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +3 -1
- package/dest/lmdb-v2/factory.d.ts +4 -7
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +3 -3
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/message.d.ts +1 -1
- package/dest/lmdb-v2/multi_map.d.ts +2 -1
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -1
- package/dest/lmdb-v2/multi_map.js +11 -0
- package/dest/lmdb-v2/read_transaction.d.ts +1 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/set.d.ts +1 -1
- package/dest/lmdb-v2/set.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.d.ts +1 -1
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/store.d.ts +1 -1
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/utils.d.ts +1 -10
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/utils.js +0 -94
- package/dest/lmdb-v2/write_transaction.d.ts +1 -1
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.js +9 -5
- package/dest/stores/index.d.ts +1 -1
- package/dest/stores/l2_tips_store.d.ts +3 -2
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +5 -3
- package/dest/utils.d.ts +1 -1
- package/package.json +10 -7
- package/src/config.ts +3 -3
- package/src/indexeddb/index.ts +2 -2
- package/src/indexeddb/multi_map.ts +12 -0
- package/src/interfaces/multi_map.ts +7 -0
- package/src/interfaces/multi_map_test_suite.ts +100 -0
- package/src/lmdb/index.ts +3 -3
- package/src/lmdb/multi_map.ts +16 -0
- package/src/lmdb/store.ts +1 -1
- package/src/lmdb-v2/factory.ts +3 -3
- package/src/lmdb-v2/multi_map.ts +9 -0
- package/src/lmdb-v2/utils.ts +0 -118
- package/src/lmdb-v2/write_transaction.ts +9 -8
- package/src/stores/l2_tips_store.ts +7 -5
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider, L2Tips } from '@aztec/stdlib/block';
|
|
2
3
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
3
4
|
/** Stores currently synced L2 tips and unfinalized block hashes. */
|
|
@@ -5,10 +6,10 @@ export declare class L2TipsKVStore implements L2BlockStreamEventHandler, L2Block
|
|
|
5
6
|
private readonly l2TipsStore;
|
|
6
7
|
private readonly l2BlockHashesStore;
|
|
7
8
|
constructor(store: AztecAsyncKVStore, namespace: string);
|
|
8
|
-
getL2BlockHash(number:
|
|
9
|
+
getL2BlockHash(number: BlockNumber): Promise<string | undefined>;
|
|
9
10
|
getL2Tips(): Promise<L2Tips>;
|
|
10
11
|
private getL2Tip;
|
|
11
12
|
handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void>;
|
|
12
13
|
private saveTag;
|
|
13
14
|
}
|
|
14
|
-
//# sourceMappingURL=
|
|
15
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibDJfdGlwc19zdG9yZS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3N0b3Jlcy9sMl90aXBzX3N0b3JlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUM5RCxPQUFPLEtBQUssRUFFVixrQkFBa0IsRUFDbEIseUJBQXlCLEVBQ3pCLDhCQUE4QixFQUU5QixNQUFNLEVBQ1AsTUFBTSxxQkFBcUIsQ0FBQztBQUc3QixPQUFPLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBRWhFLG9FQUFvRTtBQUNwRSxxQkFBYSxhQUFjLFlBQVcseUJBQXlCLEVBQUUsOEJBQThCO0lBQzdGLE9BQU8sQ0FBQyxRQUFRLENBQUMsV0FBVyxDQUF5QztJQUNyRSxPQUFPLENBQUMsUUFBUSxDQUFDLGtCQUFrQixDQUFxQztJQUV4RSxZQUFZLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxTQUFTLEVBQUUsTUFBTSxFQUd0RDtJQUVNLGNBQWMsQ0FBQyxNQUFNLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxNQUFNLEdBQUcsU0FBUyxDQUFDLENBRXRFO0lBRVksU0FBUyxJQUFJLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FNeEM7WUFFYSxRQUFRO0lBYVQsc0JBQXNCLENBQUMsS0FBSyxFQUFFLGtCQUFrQixHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0F1QjVFO1lBRWEsT0FBTztDQU10QiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"l2_tips_store.d.ts","sourceRoot":"","sources":["../../src/stores/l2_tips_store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"l2_tips_store.d.ts","sourceRoot":"","sources":["../../src/stores/l2_tips_store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAEV,kBAAkB,EAClB,yBAAyB,EACzB,8BAA8B,EAE9B,MAAM,EACP,MAAM,qBAAqB,CAAC;AAG7B,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,oEAAoE;AACpE,qBAAa,aAAc,YAAW,yBAAyB,EAAE,8BAA8B;IAC7F,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyC;IACrE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAqC;IAExE,YAAY,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,EAGtD;IAEM,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAEtE;IAEY,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,CAMxC;YAEa,QAAQ;IAaT,sBAAsB,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuB5E;YAEa,OAAO;CAMtB"}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants';
|
|
2
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
1
3
|
/** Stores currently synced L2 tips and unfinalized block hashes. */ export class L2TipsKVStore {
|
|
2
4
|
l2TipsStore;
|
|
3
5
|
l2BlockHashesStore;
|
|
@@ -25,8 +27,8 @@
|
|
|
25
27
|
const blockNumber = await this.l2TipsStore.getAsync(tag);
|
|
26
28
|
if (blockNumber === undefined || blockNumber === 0) {
|
|
27
29
|
return {
|
|
28
|
-
number:
|
|
29
|
-
hash:
|
|
30
|
+
number: BlockNumber.ZERO,
|
|
31
|
+
hash: GENESIS_BLOCK_HEADER_HASH.toString()
|
|
30
32
|
};
|
|
31
33
|
}
|
|
32
34
|
const blockHash = await this.l2BlockHashesStore.getAsync(blockNumber);
|
|
@@ -44,7 +46,7 @@
|
|
|
44
46
|
{
|
|
45
47
|
const blocks = event.blocks.map((b)=>b.block);
|
|
46
48
|
for (const block of blocks){
|
|
47
|
-
await this.l2BlockHashesStore.set(block.number, (await block.
|
|
49
|
+
await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString());
|
|
48
50
|
}
|
|
49
51
|
await this.l2TipsStore.set('latest', blocks.at(-1).number);
|
|
50
52
|
break;
|
package/dest/utils.d.ts
CHANGED
|
@@ -9,4 +9,4 @@ import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
|
|
|
9
9
|
* @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match
|
|
10
10
|
*/
|
|
11
11
|
export declare function initStoreForRollup<T extends AztecKVStore | AztecAsyncKVStore>(store: T, rollupAddress: EthAddress, log?: Logger): Promise<T>;
|
|
12
|
-
//# sourceMappingURL=
|
|
12
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidXRpbHMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy91dGlscy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxVQUFVLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxZQUFZLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUc3RTs7Ozs7O0dBTUc7QUFDSCx3QkFBc0Isa0JBQWtCLENBQUMsQ0FBQyxTQUFTLFlBQVksR0FBRyxpQkFBaUIsRUFDakYsS0FBSyxFQUFFLENBQUMsRUFDUixhQUFhLEVBQUUsVUFBVSxFQUN6QixHQUFHLENBQUMsRUFBRSxNQUFNLEdBQ1gsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQXFCWiJ9
|
package/package.json
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/kv-store",
|
|
3
|
-
"version": "4.0.0-nightly.
|
|
3
|
+
"version": "4.0.0-nightly.20260107",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./dest/interfaces/index.js",
|
|
7
|
+
"./interfaces": "./dest/interfaces/index.js",
|
|
7
8
|
"./lmdb": "./dest/lmdb/index.js",
|
|
8
9
|
"./lmdb-v2": "./dest/lmdb-v2/index.js",
|
|
9
10
|
"./indexeddb": "./dest/indexeddb/index.js",
|
|
@@ -11,8 +12,8 @@
|
|
|
11
12
|
"./config": "./dest/config.js"
|
|
12
13
|
},
|
|
13
14
|
"scripts": {
|
|
14
|
-
"build": "yarn clean && tsc
|
|
15
|
-
"build:dev": "tsc
|
|
15
|
+
"build": "yarn clean && ../scripts/tsc.sh",
|
|
16
|
+
"build:dev": "../scripts/tsc.sh --watch",
|
|
16
17
|
"clean": "rm -rf ./dest .tsbuildinfo",
|
|
17
18
|
"test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
|
|
18
19
|
"test:browser": "wtr --config ./web-test-runner.config.mjs",
|
|
@@ -24,10 +25,11 @@
|
|
|
24
25
|
"./package.local.json"
|
|
25
26
|
],
|
|
26
27
|
"dependencies": {
|
|
27
|
-
"@aztec/
|
|
28
|
-
"@aztec/
|
|
29
|
-
"@aztec/
|
|
30
|
-
"@aztec/
|
|
28
|
+
"@aztec/constants": "4.0.0-nightly.20260107",
|
|
29
|
+
"@aztec/ethereum": "4.0.0-nightly.20260107",
|
|
30
|
+
"@aztec/foundation": "4.0.0-nightly.20260107",
|
|
31
|
+
"@aztec/native": "4.0.0-nightly.20260107",
|
|
32
|
+
"@aztec/stdlib": "4.0.0-nightly.20260107",
|
|
31
33
|
"idb": "^8.0.0",
|
|
32
34
|
"lmdb": "^3.2.0",
|
|
33
35
|
"msgpackr": "^1.11.2",
|
|
@@ -43,6 +45,7 @@
|
|
|
43
45
|
"@types/mocha-each": "^2.0.4",
|
|
44
46
|
"@types/node": "^22.15.17",
|
|
45
47
|
"@types/sinon": "^17.0.3",
|
|
48
|
+
"@typescript/native-preview": "7.0.0-dev.20251126.1",
|
|
46
49
|
"@web/dev-server-esbuild": "^1.0.3",
|
|
47
50
|
"@web/test-runner": "^0.19.0",
|
|
48
51
|
"@web/test-runner-playwright": "^0.11.0",
|
package/src/config.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
|
4
4
|
|
|
5
5
|
export type DataStoreConfig = {
|
|
6
6
|
dataDirectory: string | undefined;
|
|
7
|
-
|
|
7
|
+
dataStoreMapSizeKb: number;
|
|
8
8
|
l1Contracts?: { rollupAddress: EthAddress };
|
|
9
9
|
};
|
|
10
10
|
|
|
@@ -13,9 +13,9 @@ export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
|
|
|
13
13
|
env: 'DATA_DIRECTORY',
|
|
14
14
|
description: 'Optional dir to store data. If omitted will store in memory.',
|
|
15
15
|
},
|
|
16
|
-
|
|
16
|
+
dataStoreMapSizeKb: {
|
|
17
17
|
env: 'DATA_STORE_MAP_SIZE_KB',
|
|
18
|
-
description: '
|
|
18
|
+
description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
|
|
19
19
|
...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
|
|
20
20
|
},
|
|
21
21
|
l1Contracts: {
|
package/src/indexeddb/index.ts
CHANGED
|
@@ -14,8 +14,8 @@ export async function createStore(name: string, config: DataStoreConfig, log: Lo
|
|
|
14
14
|
|
|
15
15
|
log.info(
|
|
16
16
|
dataDirectory
|
|
17
|
-
? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.
|
|
18
|
-
: `Creating ${name} ephemeral data store with map size ${config.
|
|
17
|
+
? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKb} KB`
|
|
18
|
+
: `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
|
|
19
19
|
);
|
|
20
20
|
const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
|
|
21
21
|
if (config.l1Contracts?.rollupAddress) {
|
|
@@ -61,6 +61,18 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
|
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
65
|
+
// Count entries over the keyCount index range for this key
|
|
66
|
+
const index = this.db.index('keyCount');
|
|
67
|
+
const rangeQuery = IDBKeyRange.bound(
|
|
68
|
+
[this.container, this.normalizeKey(key), 0],
|
|
69
|
+
[this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
70
|
+
false,
|
|
71
|
+
false,
|
|
72
|
+
);
|
|
73
|
+
return index.count(rangeQuery);
|
|
74
|
+
}
|
|
75
|
+
|
|
64
76
|
async deleteValue(key: K, val: V): Promise<void> {
|
|
65
77
|
// Since we know the value, we can hash it and directly query the "hash" index
|
|
66
78
|
// to avoid having to iterate over all the values
|
|
@@ -29,6 +29,13 @@ export interface AztecAsyncMultiMap<K extends Key, V extends Value> extends Azte
|
|
|
29
29
|
*/
|
|
30
30
|
getValuesAsync(key: K): AsyncIterableIterator<V>;
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Gets the number of values at the given key.
|
|
34
|
+
* @param key - The key to get the number of values from
|
|
35
|
+
* @returns The number of values at the given key
|
|
36
|
+
*/
|
|
37
|
+
getValueCountAsync(key: K): Promise<number>;
|
|
38
|
+
|
|
32
39
|
/**
|
|
33
40
|
* Deletes a specific value at the given key.
|
|
34
41
|
* @param key - The key to delete the value at
|
|
@@ -61,6 +61,13 @@ export function describeAztecMultiMap(
|
|
|
61
61
|
: await toArray((multiMap as AztecAsyncMultiMap<any, any>).getValuesAsync(key));
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
async function getValueCount(
|
|
65
|
+
key: Key,
|
|
66
|
+
sut: AztecAsyncMultiMap<any, any> | AztecMultiMap<any, any> = multiMap,
|
|
67
|
+
): Promise<number> {
|
|
68
|
+
return await (sut as AztecAsyncMultiMap<any, any>).getValueCountAsync(key);
|
|
69
|
+
}
|
|
70
|
+
|
|
64
71
|
it('should be able to set and get values', async () => {
|
|
65
72
|
await multiMap.set('foo', 'bar');
|
|
66
73
|
await multiMap.set('baz', 'qux');
|
|
@@ -238,5 +245,98 @@ export function describeAztecMultiMap(
|
|
|
238
245
|
expect(await keys({ start: 'b', reverse: true })).to.deep.equal(['d', 'c']);
|
|
239
246
|
expect(await keys({ end: 'b', reverse: true })).to.deep.equal(['b', 'a']);
|
|
240
247
|
});
|
|
248
|
+
|
|
249
|
+
it('returns 0 for missing key', async () => {
|
|
250
|
+
expect(await getValueCount('missing')).to.equal(0);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it('counts a single value', async () => {
|
|
254
|
+
await multiMap.set('foo', 'bar');
|
|
255
|
+
expect(await getValueCount('foo')).to.equal(1);
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
it('counts multiple distinct values for same key', async () => {
|
|
259
|
+
await multiMap.set('foo', 'bar');
|
|
260
|
+
await multiMap.set('foo', 'baz');
|
|
261
|
+
await multiMap.set('foo', 'qux');
|
|
262
|
+
expect(await getValueCount('foo')).to.equal(3);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
it('does not increase count for duplicate inserts', async () => {
|
|
266
|
+
await multiMap.set('foo', 'bar');
|
|
267
|
+
await multiMap.set('foo', 'bar');
|
|
268
|
+
await multiMap.set('foo', 'baz');
|
|
269
|
+
await multiMap.set('foo', 'baz');
|
|
270
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
271
|
+
expect(await getValues('foo')).to.have.members(['bar', 'baz']);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it('decrements when deleting a single value', async () => {
|
|
275
|
+
await multiMap.set('foo', '1');
|
|
276
|
+
await multiMap.set('foo', '2');
|
|
277
|
+
await multiMap.set('foo', '3');
|
|
278
|
+
expect(await getValueCount('foo')).to.equal(3);
|
|
279
|
+
await multiMap.deleteValue('foo', '2');
|
|
280
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
281
|
+
expect(await getValues('foo')).to.have.members(['1', '3']);
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
it('does not change count when deleting a non-existent value', async () => {
|
|
285
|
+
await multiMap.set('foo', '1');
|
|
286
|
+
await multiMap.set('foo', '3');
|
|
287
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
288
|
+
await multiMap.deleteValue('foo', '2');
|
|
289
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
it('clears all values when deleting a key', async () => {
|
|
293
|
+
await multiMap.set('foo', 'bar');
|
|
294
|
+
await multiMap.set('foo', 'baz');
|
|
295
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
296
|
+
await multiMap.delete('foo');
|
|
297
|
+
expect(await getValueCount('foo')).to.equal(0);
|
|
298
|
+
expect(await getValues('foo')).to.deep.equal([]);
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
it('count equals enumerated values length', async () => {
|
|
302
|
+
await multiMap.set('foo', '1');
|
|
303
|
+
await multiMap.set('foo', '2');
|
|
304
|
+
const vals = await getValues('foo');
|
|
305
|
+
expect(await getValueCount('foo')).to.equal(vals.length);
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
it('sum of per-key counts equals total size', async () => {
|
|
309
|
+
await multiMap.set('foo', '1');
|
|
310
|
+
await multiMap.set('foo', '2');
|
|
311
|
+
await multiMap.set('bar', '3');
|
|
312
|
+
await multiMap.set('bar', '4');
|
|
313
|
+
await multiMap.set('baz', '5');
|
|
314
|
+
|
|
315
|
+
const allKeys = await keys();
|
|
316
|
+
const uniqueKeys = Array.from(new Set(allKeys));
|
|
317
|
+
const counts = await Promise.all(uniqueKeys.map(k => getValueCount(k)));
|
|
318
|
+
const sum = counts.reduce((s, n) => s + n, 0);
|
|
319
|
+
expect(sum).to.equal(await size());
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
it('supports sparse slots: delete middle, reinsert new, count remains correct', async () => {
|
|
323
|
+
await multiMap.set('foo', '1');
|
|
324
|
+
await multiMap.set('foo', '2');
|
|
325
|
+
await multiMap.set('foo', '3');
|
|
326
|
+
expect(await getValueCount('foo')).to.equal(3);
|
|
327
|
+
await multiMap.deleteValue('foo', '2');
|
|
328
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
329
|
+
await multiMap.set('foo', '4');
|
|
330
|
+
expect(await getValueCount('foo')).to.equal(3);
|
|
331
|
+
expect(await getValues('foo')).to.have.members(['1', '3', '4']);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
it('multiple keys are independent', async () => {
|
|
335
|
+
await multiMap.set('foo', '1');
|
|
336
|
+
await multiMap.set('foo', '2');
|
|
337
|
+
await multiMap.set('bar', '3');
|
|
338
|
+
expect(await getValueCount('foo')).to.equal(2);
|
|
339
|
+
expect(await getValueCount('bar')).to.equal(1);
|
|
340
|
+
});
|
|
241
341
|
});
|
|
242
342
|
}
|
package/src/lmdb/index.ts
CHANGED
|
@@ -16,11 +16,11 @@ export function createStore(name: string, config: DataStoreConfig, log: Logger =
|
|
|
16
16
|
|
|
17
17
|
log.info(
|
|
18
18
|
dataDirectory
|
|
19
|
-
? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.
|
|
20
|
-
: `Creating ${name} ephemeral data store with map size ${config.
|
|
19
|
+
? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKb} KB`
|
|
20
|
+
: `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
|
|
21
21
|
);
|
|
22
22
|
|
|
23
|
-
const store = AztecLmdbStore.open(dataDirectory, config.
|
|
23
|
+
const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKb, false);
|
|
24
24
|
if (config.l1Contracts?.rollupAddress) {
|
|
25
25
|
return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
|
|
26
26
|
}
|
package/src/lmdb/multi_map.ts
CHANGED
|
@@ -29,6 +29,22 @@ export class LmdbAztecMultiMap<K extends Key, V extends Value>
|
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
33
|
+
const transaction = this.db.useReadTransaction();
|
|
34
|
+
try {
|
|
35
|
+
const values = this.db.getValues(this.slot(key), {
|
|
36
|
+
transaction,
|
|
37
|
+
});
|
|
38
|
+
let count = 0;
|
|
39
|
+
for (const _ of values) {
|
|
40
|
+
count++;
|
|
41
|
+
}
|
|
42
|
+
return Promise.resolve(count);
|
|
43
|
+
} finally {
|
|
44
|
+
transaction.done();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
32
48
|
async deleteValue(key: K, val: V): Promise<void> {
|
|
33
49
|
await this.db.remove(this.slot(key), [key, val]);
|
|
34
50
|
}
|
package/src/lmdb/store.ts
CHANGED
package/src/lmdb-v2/factory.ts
CHANGED
|
@@ -33,15 +33,15 @@ export async function createStore(
|
|
|
33
33
|
rollupAddress,
|
|
34
34
|
dataDirectory: subDir,
|
|
35
35
|
onOpen: dbDirectory =>
|
|
36
|
-
AztecLMDBStoreV2.new(dbDirectory, config.
|
|
36
|
+
AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), log),
|
|
37
37
|
});
|
|
38
38
|
|
|
39
39
|
log.info(
|
|
40
|
-
`Creating ${name} data store at directory ${subDir} with map size ${config.
|
|
40
|
+
`Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKb} KB (LMDB v2)`,
|
|
41
41
|
);
|
|
42
42
|
[store] = await versionManager.open();
|
|
43
43
|
} else {
|
|
44
|
-
store = await openTmpStore(name, true, config.
|
|
44
|
+
store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, log);
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
return store;
|
package/src/lmdb-v2/multi_map.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr/pack';
|
|
2
|
+
import { MAXIMUM_KEY, toBufferKey } from 'ordered-binary';
|
|
2
3
|
|
|
3
4
|
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
4
5
|
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
5
6
|
import type { ReadTransaction } from './read_transaction.js';
|
|
7
|
+
// eslint-disable-next-line import/no-cycle
|
|
6
8
|
import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
|
|
7
9
|
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
8
10
|
|
|
@@ -138,4 +140,11 @@ export class LMDBMultiMap<K extends Key, V extends Value> implements AztecAsyncM
|
|
|
138
140
|
yield this.encoder.unpack(value);
|
|
139
141
|
}
|
|
140
142
|
}
|
|
143
|
+
|
|
144
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
145
|
+
const startKey = serializeKey(this.prefix, key);
|
|
146
|
+
const endKey = toBufferKey([this.prefix, key, MAXIMUM_KEY]);
|
|
147
|
+
|
|
148
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(startKey, endKey, false));
|
|
149
|
+
}
|
|
141
150
|
}
|
package/src/lmdb-v2/utils.ts
CHANGED
|
@@ -2,124 +2,6 @@ import { MAXIMUM_KEY, fromBufferKey, toBufferKey } from 'ordered-binary';
|
|
|
2
2
|
|
|
3
3
|
import type { Key } from '../interfaces/common.js';
|
|
4
4
|
|
|
5
|
-
type Cmp<T> = (a: T, b: T) => -1 | 0 | 1;
|
|
6
|
-
|
|
7
|
-
export function dedupeSortedArray<T>(arr: T[], cmp: Cmp<T>): void {
|
|
8
|
-
for (let i = 0; i < arr.length; i++) {
|
|
9
|
-
let j = i + 1;
|
|
10
|
-
for (; j < arr.length; j++) {
|
|
11
|
-
const res = cmp(arr[i], arr[j]);
|
|
12
|
-
if (res === 0) {
|
|
13
|
-
continue;
|
|
14
|
-
} else if (res < 0) {
|
|
15
|
-
break;
|
|
16
|
-
} else {
|
|
17
|
-
throw new Error('Array not sorted');
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
if (j - i > 1) {
|
|
22
|
-
arr.splice(i + 1, j - i - 1);
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function insertIntoSortedArray<T>(arr: T[], item: T, cmp: (a: T, b: T) => number): void {
|
|
28
|
-
let left = 0;
|
|
29
|
-
let right = arr.length;
|
|
30
|
-
|
|
31
|
-
while (left < right) {
|
|
32
|
-
const mid = (left + right) >> 1;
|
|
33
|
-
const comparison = cmp(arr[mid], item);
|
|
34
|
-
|
|
35
|
-
if (comparison < 0) {
|
|
36
|
-
left = mid + 1;
|
|
37
|
-
} else {
|
|
38
|
-
right = mid;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
arr.splice(left, 0, item);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export function findIndexInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): number {
|
|
46
|
-
let start = 0;
|
|
47
|
-
let end = values.length - 1;
|
|
48
|
-
|
|
49
|
-
while (start <= end) {
|
|
50
|
-
const mid = start + (((end - start) / 2) | 0);
|
|
51
|
-
const res = cmp(values[mid], needle);
|
|
52
|
-
if (res === 0) {
|
|
53
|
-
return mid;
|
|
54
|
-
} else if (res > 0) {
|
|
55
|
-
end = mid - 1;
|
|
56
|
-
} else {
|
|
57
|
-
start = mid + 1;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
return -1;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export function findInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): T | undefined {
|
|
65
|
-
const idx = findIndexInSortedArray(values, needle, cmp);
|
|
66
|
-
return idx > -1 ? values[idx] : undefined;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export function removeAnyOf<T, N>(arr: T[], vals: N[], cmp: (a: T, b: N) => -1 | 0 | 1): void {
|
|
70
|
-
let writeIdx = 0;
|
|
71
|
-
let readIdx = 0;
|
|
72
|
-
let valIdx = 0;
|
|
73
|
-
|
|
74
|
-
while (readIdx < arr.length && valIdx < vals.length) {
|
|
75
|
-
const comparison = cmp(arr[readIdx], vals[valIdx]);
|
|
76
|
-
|
|
77
|
-
if (comparison < 0) {
|
|
78
|
-
arr[writeIdx++] = arr[readIdx++];
|
|
79
|
-
} else if (comparison > 0) {
|
|
80
|
-
valIdx++;
|
|
81
|
-
} else {
|
|
82
|
-
readIdx++;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
while (readIdx < arr.length) {
|
|
87
|
-
arr[writeIdx++] = arr[readIdx++];
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
arr.length = writeIdx;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export function removeFromSortedArray<T, N>(arr: T[], val: N, cmp: (a: T, b: N) => -1 | 0 | 1) {
|
|
94
|
-
const idx = findIndexInSortedArray(arr, val, cmp);
|
|
95
|
-
if (idx > -1) {
|
|
96
|
-
arr.splice(idx, 1);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export function merge<T>(arr: T[], toInsert: T[], cmp: (a: T, b: T) => -1 | 0 | 1): void {
|
|
101
|
-
const result = new Array<T>(arr.length + toInsert.length);
|
|
102
|
-
let i = 0,
|
|
103
|
-
j = 0,
|
|
104
|
-
k = 0;
|
|
105
|
-
|
|
106
|
-
while (i < arr.length && j < toInsert.length) {
|
|
107
|
-
result[k++] = cmp(arr[i], toInsert[j]) <= 0 ? arr[i++] : toInsert[j++];
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
while (i < arr.length) {
|
|
111
|
-
result[k++] = arr[i++];
|
|
112
|
-
}
|
|
113
|
-
while (j < toInsert.length) {
|
|
114
|
-
result[k++] = toInsert[j++];
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
for (i = 0; i < result.length; i++) {
|
|
118
|
-
arr[i] = result[i];
|
|
119
|
-
}
|
|
120
|
-
arr.length = result.length;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
5
|
export function keyCmp(a: [Uint8Array, Uint8Array[] | null], b: [Uint8Array, Uint8Array[] | null]): -1 | 0 | 1 {
|
|
124
6
|
return Buffer.compare(a[0], b[0]);
|
|
125
7
|
}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { type Batch, Database, LMDBMessageType } from './message.js';
|
|
2
|
-
import { ReadTransaction } from './read_transaction.js';
|
|
3
1
|
import {
|
|
4
2
|
dedupeSortedArray,
|
|
5
3
|
findInSortedArray,
|
|
6
4
|
findIndexInSortedArray,
|
|
7
5
|
insertIntoSortedArray,
|
|
8
|
-
keyCmp,
|
|
9
6
|
merge,
|
|
10
7
|
removeAnyOf,
|
|
11
8
|
removeFromSortedArray,
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
} from '@aztec/foundation/array';
|
|
10
|
+
|
|
11
|
+
import { type Batch, Database, LMDBMessageType } from './message.js';
|
|
12
|
+
import { ReadTransaction } from './read_transaction.js';
|
|
13
|
+
import { keyCmp, singleKeyCmp } from './utils.js';
|
|
14
14
|
|
|
15
15
|
export class WriteTransaction extends ReadTransaction {
|
|
16
16
|
// exposed for tests
|
|
@@ -44,7 +44,7 @@ export class WriteTransaction extends ReadTransaction {
|
|
|
44
44
|
remove(key: Uint8Array): Promise<void> {
|
|
45
45
|
const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
|
|
46
46
|
if (removeEntryIndex === -1) {
|
|
47
|
-
this.dataBatch.removeEntries
|
|
47
|
+
insertIntoSortedArray(this.dataBatch.removeEntries, [key, null], keyCmp);
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
const addEntryIndex = findIndexInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
|
|
@@ -218,8 +218,9 @@ export class WriteTransaction extends ReadTransaction {
|
|
|
218
218
|
): AsyncIterable<[Uint8Array, T]> {
|
|
219
219
|
this.assertIsOpen();
|
|
220
220
|
|
|
221
|
-
//
|
|
221
|
+
// Snapshot both add and remove entries at the start of iteration to ensure consistency
|
|
222
222
|
const uncommittedEntries = [...batch.addEntries];
|
|
223
|
+
const removeEntries = [...batch.removeEntries];
|
|
223
224
|
// used to check we're in the right order when comparing between a key and uncommittedEntries
|
|
224
225
|
let cmpDirection = -1;
|
|
225
226
|
if (reverse) {
|
|
@@ -262,7 +263,7 @@ export class WriteTransaction extends ReadTransaction {
|
|
|
262
263
|
break;
|
|
263
264
|
}
|
|
264
265
|
|
|
265
|
-
const toRemove = findInSortedArray(
|
|
266
|
+
const toRemove = findInSortedArray(removeEntries, key, singleKeyCmp);
|
|
266
267
|
|
|
267
268
|
// at this point we've either exhausted all uncommitted entries,
|
|
268
269
|
// we reached a key strictly greater/smaller than `key`
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants';
|
|
2
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
1
3
|
import type {
|
|
2
4
|
L2BlockId,
|
|
3
5
|
L2BlockStreamEvent,
|
|
@@ -12,15 +14,15 @@ import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
|
12
14
|
|
|
13
15
|
/** Stores currently synced L2 tips and unfinalized block hashes. */
|
|
14
16
|
export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
|
|
15
|
-
private readonly l2TipsStore: AztecAsyncMap<L2BlockTag,
|
|
16
|
-
private readonly l2BlockHashesStore: AztecAsyncMap<
|
|
17
|
+
private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, BlockNumber>;
|
|
18
|
+
private readonly l2BlockHashesStore: AztecAsyncMap<BlockNumber, string>;
|
|
17
19
|
|
|
18
20
|
constructor(store: AztecAsyncKVStore, namespace: string) {
|
|
19
21
|
this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_'));
|
|
20
22
|
this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_'));
|
|
21
23
|
}
|
|
22
24
|
|
|
23
|
-
public getL2BlockHash(number:
|
|
25
|
+
public getL2BlockHash(number: BlockNumber): Promise<string | undefined> {
|
|
24
26
|
return this.l2BlockHashesStore.getAsync(number);
|
|
25
27
|
}
|
|
26
28
|
|
|
@@ -35,7 +37,7 @@ export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLo
|
|
|
35
37
|
private async getL2Tip(tag: L2BlockTag): Promise<L2BlockId> {
|
|
36
38
|
const blockNumber = await this.l2TipsStore.getAsync(tag);
|
|
37
39
|
if (blockNumber === undefined || blockNumber === 0) {
|
|
38
|
-
return { number:
|
|
40
|
+
return { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() };
|
|
39
41
|
}
|
|
40
42
|
const blockHash = await this.l2BlockHashesStore.getAsync(blockNumber);
|
|
41
43
|
if (!blockHash) {
|
|
@@ -50,7 +52,7 @@ export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLo
|
|
|
50
52
|
case 'blocks-added': {
|
|
51
53
|
const blocks = event.blocks.map(b => b.block);
|
|
52
54
|
for (const block of blocks) {
|
|
53
|
-
await this.l2BlockHashesStore.set(block.number, (await block.
|
|
55
|
+
await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString());
|
|
54
56
|
}
|
|
55
57
|
await this.l2TipsStore.set('latest', blocks.at(-1)!.number);
|
|
56
58
|
break;
|