@typeberry/lib 0.11.0-829b7ea → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -1
- package/packages/jam/config-node/node-config.d.ts +11 -1
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +25 -3
- package/packages/jam/config-node/node-config.test.js +7 -3
- package/packages/jam/database/states.d.ts +8 -1
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database/value-refs.d.ts +5 -4
- package/packages/jam/database/value-refs.d.ts.map +1 -1
- package/packages/jam/database-lmdb/blocks.d.ts +24 -0
- package/packages/jam/database-lmdb/blocks.d.ts.map +1 -0
- package/packages/jam/database-lmdb/blocks.js +82 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +56 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -0
- package/packages/jam/database-lmdb/hybrid-states.js +149 -0
- package/packages/jam/database-lmdb/hybrid-states.test.d.ts +2 -0
- package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map +1 -0
- package/packages/jam/database-lmdb/hybrid-states.test.js +180 -0
- package/packages/jam/database-lmdb/index.d.ts +5 -0
- package/packages/jam/database-lmdb/index.d.ts.map +1 -0
- package/packages/jam/database-lmdb/index.js +4 -0
- package/packages/jam/database-lmdb/root.d.ts +26 -0
- package/packages/jam/database-lmdb/root.d.ts.map +1 -0
- package/packages/jam/database-lmdb/root.js +53 -0
- package/packages/jam/database-lmdb/states.d.ts +76 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -0
- package/packages/jam/database-lmdb/states.js +145 -0
- package/packages/jam/database-lmdb/states.test.d.ts +2 -0
- package/packages/jam/database-lmdb/states.test.d.ts.map +1 -0
- package/packages/jam/database-lmdb/states.test.js +202 -0
- package/packages/jam/node/export.d.ts.map +1 -1
- package/packages/jam/node/export.js +17 -8
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +5 -3
- package/packages/jam/node/main-importer.d.ts +6 -2
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +27 -12
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +16 -6
- package/packages/jam/rpc/test/e2e-server.d.ts.map +1 -1
- package/packages/jam/rpc/test/e2e-server.js +5 -3
- package/packages/jam/rpc/test/e2e.js +1 -1
- package/packages/jam/state-merkleization/index.d.ts +1 -1
- package/packages/jam/state-merkleization/index.js +1 -1
- package/packages/workers/api-node/config.d.ts +40 -6
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +82 -6
- package/packages/workers/api-node/config.test.js +63 -26
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { afterEach, before, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
|
|
5
|
+
import { Bytes, BytesBlob } from "#@typeberry/bytes";
|
|
6
|
+
import { SortedSet } from "#@typeberry/collections";
|
|
7
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
8
|
+
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
9
|
+
import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
|
|
10
|
+
import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
|
|
11
|
+
import { testState } from "#@typeberry/state/test.utils.js";
|
|
12
|
+
import { StateEntries } from "#@typeberry/state-merkleization";
|
|
13
|
+
import { InMemoryTrie, leafComparator } from "#@typeberry/trie";
|
|
14
|
+
import { getBlake2bTrieHasher } from "#@typeberry/trie/hasher.js";
|
|
15
|
+
import { deepEqual, OK, Result } from "#@typeberry/utils";
|
|
16
|
+
import { LmdbRoot } from "./root.js";
|
|
17
|
+
import { LmdbStates } from "./states.js";
|
|
18
|
+
let blake2bTrieHasher;
|
|
19
|
+
let blake2b;
|
|
20
|
+
before(async () => {
|
|
21
|
+
blake2b = await Blake2b.createHasher();
|
|
22
|
+
blake2bTrieHasher = getBlake2bTrieHasher(blake2b);
|
|
23
|
+
});
|
|
24
|
+
function createTempDir(suffix = "lmdb") {
|
|
25
|
+
return fs.mkdtempSync(`typeberry-${suffix}`);
|
|
26
|
+
}
|
|
27
|
+
describe("LMDB States database", () => {
|
|
28
|
+
let tmpDir = "";
|
|
29
|
+
beforeEach(() => {
|
|
30
|
+
tmpDir = createTempDir();
|
|
31
|
+
});
|
|
32
|
+
afterEach(() => {
|
|
33
|
+
fs.rmSync(tmpDir, {
|
|
34
|
+
recursive: true,
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
38
|
+
const spec = tinyChainSpec;
|
|
39
|
+
it("should import state and read state", async () => {
|
|
40
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
41
|
+
const states = LmdbStates.new(spec, blake2b, root);
|
|
42
|
+
try {
|
|
43
|
+
const emptyState = InMemoryState.empty(spec);
|
|
44
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
|
|
45
|
+
const emptyRoot = serialized.getRootHash(blake2b);
|
|
46
|
+
// when
|
|
47
|
+
const res = await states.insertInitialState(headerHash, serialized);
|
|
48
|
+
deepEqual(res, Result.ok(OK));
|
|
49
|
+
const newState = states.getState(headerHash);
|
|
50
|
+
assert.ok(newState !== null);
|
|
51
|
+
const newRoot = await states.getStateRoot(newState);
|
|
52
|
+
assert.deepStrictEqual(`${newRoot}`, `${emptyRoot}`);
|
|
53
|
+
deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await states.close();
|
|
57
|
+
await root.close();
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
it("should update the state", async () => {
|
|
61
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
62
|
+
const states = LmdbStates.new(spec, blake2b, root);
|
|
63
|
+
try {
|
|
64
|
+
const state = InMemoryState.empty(spec);
|
|
65
|
+
await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
|
|
66
|
+
const newState = states.getState(headerHash);
|
|
67
|
+
assert.ok(newState !== null);
|
|
68
|
+
const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
69
|
+
const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
|
|
70
|
+
const stateUpdate = {
|
|
71
|
+
timeslot: tryAsTimeSlot(15),
|
|
72
|
+
privilegedServices: PrivilegedServices.create({
|
|
73
|
+
manager: tryAsServiceId(1),
|
|
74
|
+
assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
|
|
75
|
+
delegator: tryAsServiceId(3),
|
|
76
|
+
registrar: tryAsServiceId(4),
|
|
77
|
+
autoAccumulateServices: new Map(),
|
|
78
|
+
}),
|
|
79
|
+
updated: new Map([
|
|
80
|
+
[
|
|
81
|
+
tryAsServiceId(1),
|
|
82
|
+
UpdateService.create({
|
|
83
|
+
serviceInfo: ServiceAccountInfo.create({
|
|
84
|
+
codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
|
|
85
|
+
balance: tryAsU64(1_000_000),
|
|
86
|
+
accumulateMinGas: tryAsServiceGas(10_000),
|
|
87
|
+
onTransferMinGas: tryAsServiceGas(5_000),
|
|
88
|
+
storageUtilisationBytes: tryAsU64(1_000),
|
|
89
|
+
gratisStorage: tryAsU64(0),
|
|
90
|
+
storageUtilisationCount: tryAsU32(1),
|
|
91
|
+
created: tryAsTimeSlot(0),
|
|
92
|
+
lastAccumulation: tryAsTimeSlot(0),
|
|
93
|
+
parentService: tryAsServiceId(0),
|
|
94
|
+
}),
|
|
95
|
+
lookupHistory,
|
|
96
|
+
}),
|
|
97
|
+
],
|
|
98
|
+
]),
|
|
99
|
+
};
|
|
100
|
+
// when
|
|
101
|
+
// in-memory state update
|
|
102
|
+
const res1 = state.applyUpdate(stateUpdate);
|
|
103
|
+
deepEqual(res1, Result.ok(OK));
|
|
104
|
+
// on-disk state update
|
|
105
|
+
const res2 = await states.updateAndSetState(headerHash2, newState, stateUpdate);
|
|
106
|
+
deepEqual(res2, Result.ok(OK));
|
|
107
|
+
const updatedState = states.getState(headerHash2);
|
|
108
|
+
assert.ok(updatedState !== null);
|
|
109
|
+
const updatedStateRoot = await states.getStateRoot(updatedState);
|
|
110
|
+
deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
|
|
111
|
+
[
|
|
112
|
+
tryAsServiceId(1),
|
|
113
|
+
{
|
|
114
|
+
storageKeys: [],
|
|
115
|
+
preimages: [],
|
|
116
|
+
lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
|
|
117
|
+
},
|
|
118
|
+
],
|
|
119
|
+
])), state);
|
|
120
|
+
assert.strictEqual(`${updatedStateRoot}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
await states.close();
|
|
124
|
+
await root.close();
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
it("sorted set should be actual to trie", () => {
|
|
128
|
+
const data = [
|
|
129
|
+
[Bytes.fill(HASH_SIZE, 5), BytesBlob.blobFromString("five")],
|
|
130
|
+
[Bytes.fill(HASH_SIZE, 1), BytesBlob.blobFromString("one")],
|
|
131
|
+
[Bytes.fill(HASH_SIZE, 3), BytesBlob.blobFromString("three")],
|
|
132
|
+
[Bytes.fill(HASH_SIZE, 4), BytesBlob.blobFromString("four")],
|
|
133
|
+
[Bytes.fill(HASH_SIZE, 2), BytesBlob.blobFromString("two")],
|
|
134
|
+
];
|
|
135
|
+
const trie = InMemoryTrie.empty(blake2bTrieHasher);
|
|
136
|
+
for (const [key, val] of data) {
|
|
137
|
+
trie.set(key.asOpaque(), val);
|
|
138
|
+
}
|
|
139
|
+
const set = SortedSet.fromArray(leafComparator, data.map(([key, value]) => {
|
|
140
|
+
return InMemoryTrie.constructLeaf(blake2bTrieHasher, key.asOpaque(), value);
|
|
141
|
+
}));
|
|
142
|
+
deepEqual(Array.from(set), Array.from(SortedSet.fromArray(leafComparator, Array.from(trie.nodes.leaves()))));
|
|
143
|
+
});
|
|
144
|
+
it("should import more complex state", async () => {
|
|
145
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
146
|
+
const states = LmdbStates.new(spec, blake2b, root);
|
|
147
|
+
try {
|
|
148
|
+
const initialState = testState();
|
|
149
|
+
const initialService = initialState.services.get(tryAsServiceId(0));
|
|
150
|
+
if (initialService === undefined) {
|
|
151
|
+
throw new Error("Expected service in test state!");
|
|
152
|
+
}
|
|
153
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
|
|
154
|
+
const initialRoot = serialized.getRootHash(blake2b);
|
|
155
|
+
// when
|
|
156
|
+
const res = await states.insertInitialState(headerHash, serialized);
|
|
157
|
+
deepEqual(res, Result.ok(OK));
|
|
158
|
+
const newState = states.getState(headerHash);
|
|
159
|
+
assert.ok(newState !== null);
|
|
160
|
+
const newRoot = await states.getStateRoot(newState);
|
|
161
|
+
assert.deepStrictEqual(`${newRoot}`, `${initialRoot}`);
|
|
162
|
+
deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
|
|
163
|
+
}
|
|
164
|
+
finally {
|
|
165
|
+
await states.close();
|
|
166
|
+
await root.close();
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
it("should update more complex entries", async () => {
|
|
170
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
171
|
+
const states = LmdbStates.new(spec, blake2b, root);
|
|
172
|
+
try {
|
|
173
|
+
const state = testState();
|
|
174
|
+
const initialService = state.services.get(tryAsServiceId(0));
|
|
175
|
+
if (initialService === undefined) {
|
|
176
|
+
throw new Error("Expected service in test state!");
|
|
177
|
+
}
|
|
178
|
+
await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
|
|
179
|
+
const newState = states.getState(headerHash);
|
|
180
|
+
assert.ok(newState !== null);
|
|
181
|
+
const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
182
|
+
// attempt to update all entries
|
|
183
|
+
const stateUpdate = Object.assign({}, state);
|
|
184
|
+
// when
|
|
185
|
+
// in-memory state update
|
|
186
|
+
const res1 = state.applyUpdate(stateUpdate);
|
|
187
|
+
deepEqual(res1, Result.ok(OK));
|
|
188
|
+
// on-disk state update
|
|
189
|
+
const res2 = await states.updateAndSetState(headerHash2, newState, stateUpdate);
|
|
190
|
+
deepEqual(res2, Result.ok(OK));
|
|
191
|
+
const updatedState = states.getState(headerHash2);
|
|
192
|
+
assert.ok(updatedState !== null);
|
|
193
|
+
const updatedStateRoot = await states.getStateRoot(updatedState);
|
|
194
|
+
deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([[initialService.serviceId, initialService.getEntries()]])), state);
|
|
195
|
+
assert.strictEqual(`${updatedStateRoot}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
198
|
+
await states.close();
|
|
199
|
+
await root.close();
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}
|
|
@@ -3,9 +3,10 @@ import path from "node:path";
|
|
|
3
3
|
import { Block as BlockCodec } from "#@typeberry/block";
|
|
4
4
|
import { Bytes } from "#@typeberry/bytes";
|
|
5
5
|
import { Encoder } from "#@typeberry/codec";
|
|
6
|
+
import { RegularStateBackend } from "#@typeberry/config-node";
|
|
6
7
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
7
8
|
import { Logger } from "#@typeberry/logger";
|
|
8
|
-
import { FjallWorkerConfig } from "#@typeberry/workers-api-node";
|
|
9
|
+
import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
|
|
9
10
|
import { getChainSpec, getDatabasePath } from "./common.js";
|
|
10
11
|
export async function exportBlocks(jamNodeConfig, output, withRelPath) {
|
|
11
12
|
const logger = Logger.new(import.meta.filename, "export");
|
|
@@ -29,13 +30,21 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
|
|
|
29
30
|
const blake2b = await Blake2b.createHasher();
|
|
30
31
|
const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
|
|
31
32
|
const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
|
|
32
|
-
const config =
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
33
|
+
const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
|
|
34
|
+
? FjallWorkerConfig.new({
|
|
35
|
+
nodeName: jamNodeConfig.nodeName,
|
|
36
|
+
chainSpec,
|
|
37
|
+
blake2b,
|
|
38
|
+
dbPath,
|
|
39
|
+
workerParams: undefined,
|
|
40
|
+
})
|
|
41
|
+
: LmdbWorkerConfig.new({
|
|
42
|
+
nodeName: jamNodeConfig.nodeName,
|
|
43
|
+
chainSpec,
|
|
44
|
+
blake2b,
|
|
45
|
+
dbPath,
|
|
46
|
+
workerParams: undefined,
|
|
47
|
+
});
|
|
39
48
|
const rootDb = await config.openDatabase();
|
|
40
49
|
const blocks = rootDb.getBlocksDb();
|
|
41
50
|
logger.info `📖 Gathering blocks...`;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAsBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8LrH"}
|
|
@@ -15,7 +15,8 @@ const logger = Logger.new(import.meta.filename, "fuzztarget");
|
|
|
15
15
|
/** Dedicated subdirectory under the configured base path that the fuzzer owns and wipes. */
|
|
16
16
|
const FUZZ_DB_SUBDIR = "typeberry-fuzz-db";
|
|
17
17
|
const FUZZ_DB_FJALL = "fjall-hybrid";
|
|
18
|
-
const
|
|
18
|
+
const FUZZ_DB_LMDB = "lmdb-hybrid";
|
|
19
|
+
const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, FUZZ_DB_LMDB, "fjall", "lmdb"];
|
|
19
20
|
/** Subdirectory (under the fuzzer's db dir) holding the reused fjall values keyspace. */
|
|
20
21
|
const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
|
|
21
22
|
/**
|
|
@@ -63,6 +64,7 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
63
64
|
const { jamNodeConfig: config } = fuzzConfig;
|
|
64
65
|
const fuzzDbBase = resolveFuzzDbBase(config.node.databaseBasePath);
|
|
65
66
|
const rawFuzzDb = process.env.JAM_FUZZ_DB?.trim() ?? "";
|
|
67
|
+
// Using experimental fjall-hybrid by default, with an option to test lmdb as well.
|
|
66
68
|
const hybridStateBackend = rawFuzzDb === "" ? FUZZ_DB_FJALL : rawFuzzDb;
|
|
67
69
|
if (!isValidStateBackend(hybridStateBackend)) {
|
|
68
70
|
throw new Error(`JAM_FUZZ_DB must be one of: ${FUZZ_DB_OPTIONS} (got: "${rawFuzzDb}").`);
|
|
@@ -142,7 +144,7 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
142
144
|
// also turns on compression further down, so the big values do not grow the
|
|
143
145
|
// db too much. Tiny stays uncompressed, its db is small and speed matters more.
|
|
144
146
|
ephemeral: isPersistent,
|
|
145
|
-
stateBackend: isPersistent ? hybridStateBackend : "
|
|
147
|
+
stateBackend: isPersistent ? hybridStateBackend : "lmdb",
|
|
146
148
|
// Reuse the session keyspace (fjall-hybrid only, other backends
|
|
147
149
|
// ignore it). Nothing to pass for the in-memory fallback.
|
|
148
150
|
sharedFjallSession: hybridStateBackend === "fjall-hybrid" ? (fjallSession ?? undefined) : undefined,
|
|
@@ -180,7 +182,7 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
180
182
|
}
|
|
181
183
|
}
|
|
182
184
|
else {
|
|
183
|
-
//
|
|
185
|
+
// All other backends ("fjall", "lmdb", "lmdb-hybrid"): wipe and reopen on every reset.
|
|
184
186
|
await wipeFuzzDb(fuzzDbBase);
|
|
185
187
|
}
|
|
186
188
|
runningNode = await buildNode(fuzzDbBase);
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import { type FjallValuesSession } from "#@typeberry/workers-api-node";
|
|
2
2
|
import type { JamConfig } from "./jam-config.js";
|
|
3
3
|
import type { NodeApi } from "./main.js";
|
|
4
|
-
export type StateBackend = "fjall" | "fjall-hybrid";
|
|
4
|
+
export type StateBackend = "lmdb" | "fjall" | "lmdb-hybrid" | "fjall-hybrid";
|
|
5
5
|
export type ImporterOptions = {
|
|
6
6
|
initGenesisFromAncestry?: boolean;
|
|
7
7
|
dummyFinalityDepth?: number;
|
|
8
8
|
pruneBlocks?: boolean;
|
|
9
9
|
/** Open the database without fsync/compression. Only safe for throwaway dbs (e.g. fuzzing). */
|
|
10
10
|
ephemeral?: boolean;
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* Persistent backend used when `databaseBasePath` is set. Defaults to config's backend (fjall unless overridden).
|
|
13
|
+
*
|
|
14
|
+
* lmdb and lmdb-hybrid are deprecated and retained as explicit fallbacks.
|
|
15
|
+
*/
|
|
12
16
|
stateBackend?: StateBackend;
|
|
13
17
|
/**
|
|
14
18
|
* Reuse an already-open fjall values session instead of opening a fresh
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,
|
|
1
|
+
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAoHlB"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { Bytes } from "#@typeberry/bytes";
|
|
2
2
|
import { PvmBackend } from "#@typeberry/config";
|
|
3
|
-
import { KnownChainSpec } from "#@typeberry/config-node";
|
|
3
|
+
import { KnownChainSpec, RegularStateBackend } from "#@typeberry/config-node";
|
|
4
4
|
import { bandersnatch, initWasm } from "#@typeberry/crypto";
|
|
5
5
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
6
6
|
import { createImporter, ImporterConfig } from "#@typeberry/importer";
|
|
7
7
|
import { tryAsU16 } from "#@typeberry/numbers";
|
|
8
8
|
import { CURRENT_SUITE, CURRENT_VERSION, Result, resultToString, version } from "#@typeberry/utils";
|
|
9
|
-
import { FjallWorkerConfig, HybridWorkerConfig, InMemWorkerConfig, } from "#@typeberry/workers-api-node";
|
|
9
|
+
import { FjallWorkerConfig, HybridWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, } from "#@typeberry/workers-api-node";
|
|
10
10
|
import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
|
|
11
11
|
const zeroHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
12
12
|
export async function mainImporter(config, withRelPath, options = {}) {
|
|
@@ -18,8 +18,13 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
18
18
|
logger.info `🐇 Bandersnatch ${bandesnatchNative.isOk ? "native 🚀" : `using wasm: ${bandesnatchNative.error}`}`;
|
|
19
19
|
// Single source of truth for the states db backend: drives both the log line
|
|
20
20
|
// below and the worker config picked further down.
|
|
21
|
-
const dbBackend = config.node.databaseBasePath === undefined
|
|
21
|
+
const dbBackend = config.node.databaseBasePath === undefined
|
|
22
|
+
? "in-memory"
|
|
23
|
+
: (options.stateBackend ?? config.node.stateBackend ?? RegularStateBackend.Fjall);
|
|
22
24
|
logger.info `🗄️ States DB: ${dbBackend}.`;
|
|
25
|
+
if (dbBackend === "lmdb" || dbBackend === "lmdb-hybrid") {
|
|
26
|
+
logger.warn `🗄️ The ${dbBackend} state backend is deprecated. Use fjall unless you need a temporary fallback.`;
|
|
27
|
+
}
|
|
23
28
|
const chainSpec = getChainSpec(config.node.flavor);
|
|
24
29
|
const blake2b = await Blake2b.createHasher();
|
|
25
30
|
const nodeName = config.nodeName;
|
|
@@ -39,7 +44,7 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
39
44
|
blake2b,
|
|
40
45
|
workerParams,
|
|
41
46
|
})
|
|
42
|
-
: dbBackend === "fjall-hybrid"
|
|
47
|
+
: dbBackend === "lmdb-hybrid" || dbBackend === "fjall-hybrid"
|
|
43
48
|
? await HybridWorkerConfig.new({
|
|
44
49
|
nodeName,
|
|
45
50
|
chainSpec,
|
|
@@ -48,16 +53,26 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
48
53
|
workerParams,
|
|
49
54
|
ephemeral,
|
|
50
55
|
compression,
|
|
56
|
+
backend: dbBackend === "lmdb-hybrid" ? "lmdb" : "fjall",
|
|
51
57
|
sharedFjallSession: options.sharedFjallSession,
|
|
52
58
|
})
|
|
53
|
-
:
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
: dbBackend === "fjall"
|
|
60
|
+
? FjallWorkerConfig.new({
|
|
61
|
+
nodeName,
|
|
62
|
+
chainSpec,
|
|
63
|
+
blake2b,
|
|
64
|
+
dbPath,
|
|
65
|
+
workerParams,
|
|
66
|
+
ephemeral,
|
|
67
|
+
})
|
|
68
|
+
: LmdbWorkerConfig.new({
|
|
69
|
+
nodeName,
|
|
70
|
+
chainSpec,
|
|
71
|
+
blake2b,
|
|
72
|
+
dbPath,
|
|
73
|
+
workerParams,
|
|
74
|
+
ephemeral,
|
|
75
|
+
});
|
|
61
76
|
// Initialize the database with genesis state and block if there isn't one.
|
|
62
77
|
logger.info `🛢️ Opening database at ${dbPath}`;
|
|
63
78
|
const rootDb = await workerConfig.openDatabase({ readonly: false });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAgB/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAYnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA2LlB"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isMainThread } from "node:worker_threads";
|
|
2
2
|
import { AUTHORSHIP_NETWORK_PORT } from "#@typeberry/comms-authorship-network";
|
|
3
3
|
import { PvmBackend } from "#@typeberry/config";
|
|
4
|
+
import { RegularStateBackend } from "#@typeberry/config-node";
|
|
4
5
|
import { initWasm } from "#@typeberry/crypto";
|
|
5
6
|
import { deriveBandersnatchSecretKey, deriveEd25519SecretKey, trivialSeed, } from "#@typeberry/crypto/key-derivation.js";
|
|
6
7
|
import { Blake2b } from "#@typeberry/hash";
|
|
@@ -10,7 +11,7 @@ import { Listener } from "#@typeberry/listener";
|
|
|
10
11
|
import { tryAsU16, tryAsU32 } from "#@typeberry/numbers";
|
|
11
12
|
import { CURRENT_SUITE, CURRENT_VERSION, Result, version } from "#@typeberry/utils";
|
|
12
13
|
import { DirectPort, DirectWorkerConfig } from "#@typeberry/workers-api";
|
|
13
|
-
import { FjallWorkerConfig, InMemWorkerConfig, logHostEnvironment, ThreadPort, } from "#@typeberry/workers-api-node";
|
|
14
|
+
import { FjallWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort, } from "#@typeberry/workers-api-node";
|
|
14
15
|
import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
|
|
15
16
|
import { initializeExtensions } from "./extensions.js";
|
|
16
17
|
import * as metrics from "./metrics.js";
|
|
@@ -29,9 +30,12 @@ export async function main(config, withRelPath, telemetry) {
|
|
|
29
30
|
const blake2b = await Blake2b.createHasher();
|
|
30
31
|
const nodeName = config.nodeName;
|
|
31
32
|
const isInMemory = config.node.databaseBasePath === undefined;
|
|
32
|
-
logger.info `🗄️ States DB: ${isInMemory ? "in-memory" :
|
|
33
|
+
logger.info `🗄️ States DB: ${isInMemory ? "in-memory" : config.node.stateBackend}.`;
|
|
34
|
+
if (!isInMemory && config.node.stateBackend === RegularStateBackend.Lmdb) {
|
|
35
|
+
logger.warn `🗄️ The lmdb state backend is deprecated. Use state_backend="fjall" unless you need a temporary fallback.`;
|
|
36
|
+
}
|
|
33
37
|
const { dbPath, genesisHeaderHash } = getDatabasePath(blake2b, nodeName, config.node.chainSpec.genesisHeader, withRelPath(config.node.databaseBasePath ?? "<in-memory>"));
|
|
34
|
-
const baseConfig = { nodeName, chainSpec, blake2b, dbPath };
|
|
38
|
+
const baseConfig = { nodeName, chainSpec, blake2b, dbPath, stateBackend: config.node.stateBackend };
|
|
35
39
|
const importerParams = {
|
|
36
40
|
...baseConfig,
|
|
37
41
|
workerParams: ImporterConfig.create({
|
|
@@ -59,8 +63,9 @@ export async function main(config, withRelPath, telemetry) {
|
|
|
59
63
|
throw e;
|
|
60
64
|
}
|
|
61
65
|
// fjall-js shares the engine explicitly and requires every handle to close.
|
|
66
|
+
// Keep lmdb's historical main-thread handle open until shutdown.
|
|
62
67
|
let mainRootDb = rootDb;
|
|
63
|
-
if (!importerConfig.isInMemory) {
|
|
68
|
+
if (!importerConfig.isInMemory && config.node.stateBackend === RegularStateBackend.Fjall) {
|
|
64
69
|
await rootDb.close();
|
|
65
70
|
mainRootDb = null;
|
|
66
71
|
}
|
|
@@ -236,6 +241,11 @@ const initNetwork = async (importer, params, baseConfig, genesisHeaderHash, netw
|
|
|
236
241
|
});
|
|
237
242
|
return { closeNetwork: finish, networkApi: network, networkWorker: worker };
|
|
238
243
|
};
|
|
239
|
-
function createPersistentWorkerConfig(params) {
|
|
240
|
-
|
|
244
|
+
function createPersistentWorkerConfig({ stateBackend, ...params }) {
|
|
245
|
+
switch (stateBackend) {
|
|
246
|
+
case RegularStateBackend.Fjall:
|
|
247
|
+
return FjallWorkerConfig.new(params);
|
|
248
|
+
case RegularStateBackend.Lmdb:
|
|
249
|
+
return LmdbWorkerConfig.new(params);
|
|
250
|
+
}
|
|
241
251
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"e2e-server.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/rpc/test/e2e-server.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAM7C,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,SAAe,
|
|
1
|
+
{"version":3,"file":"e2e-server.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/rpc/test/e2e-server.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAM7C,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,SAAe,sBA6B/E"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PvmBackend } from "#@typeberry/config";
|
|
2
|
-
import { loadConfig, NODE_DEFAULTS } from "#@typeberry/config-node";
|
|
2
|
+
import { loadConfig, NODE_DEFAULTS, RegularStateBackend } from "#@typeberry/config-node";
|
|
3
3
|
import { Blake2b } from "#@typeberry/hash";
|
|
4
4
|
import { getChainSpec, getDatabasePath } from "#@typeberry/node";
|
|
5
5
|
import { validation } from "#@typeberry/rpc-validation";
|
|
6
6
|
import { workspacePathFix } from "#@typeberry/utils";
|
|
7
|
-
import { FjallWorkerConfig } from "#@typeberry/workers-api-node";
|
|
7
|
+
import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
|
|
8
8
|
import { handlers } from "../src/handlers.js";
|
|
9
9
|
import { RpcServer } from "../src/server.js";
|
|
10
10
|
const DEFAULT_PORT = 19800;
|
|
@@ -25,6 +25,8 @@ export async function startTestRpcServer(configPath, port = DEFAULT_PORT) {
|
|
|
25
25
|
dbPath,
|
|
26
26
|
blake2b,
|
|
27
27
|
};
|
|
28
|
-
const rootDb =
|
|
28
|
+
const rootDb = nodeConfig.stateBackend === RegularStateBackend.Fjall
|
|
29
|
+
? await FjallWorkerConfig.new(dbConfigParams).openDatabase({ readonly: true })
|
|
30
|
+
: await LmdbWorkerConfig.new(dbConfigParams).openDatabase({ readonly: true });
|
|
29
31
|
return RpcServer.new(port, rootDb, spec, blake2b, PvmBackend.Ananas, handlers, validation.schemas);
|
|
30
32
|
}
|
|
@@ -94,7 +94,7 @@ describe("JSON RPC Client-Server E2E", { concurrency: false }, () => {
|
|
|
94
94
|
// TODO [ToDr] We should probably do a little bit better in terms of
|
|
95
95
|
// tracking recently active services. Some options for the future:
|
|
96
96
|
// 1. Use InMemoryDb for RPC E2E tests.
|
|
97
|
-
// 2. Store additional service metadata in
|
|
97
|
+
// 2. Store additional service metadata in LMDB
|
|
98
98
|
// 3. Cache the state object, so that accessed services would be returned here.
|
|
99
99
|
assert.deepStrictEqual(result, []);
|
|
100
100
|
});
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* (incomplete in-memory view).
|
|
20
20
|
* - `SerializedState<LeafDb>`: Disk-backed trie storage-leaf nodes live on
|
|
21
21
|
* disk and load on demand; cheap to update (no data duplication) and re-compute
|
|
22
|
-
* the `stateRoot`.
|
|
22
|
+
* the `stateRoot`. Used in LMDB.
|
|
23
23
|
* - `SerializedState<StateEntries>`: serialized state represented as a simple in-memory
|
|
24
24
|
* hashmap of `key -> value` entries.
|
|
25
25
|
*/
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* (incomplete in-memory view).
|
|
20
20
|
* - `SerializedState<LeafDb>`: Disk-backed trie storage-leaf nodes live on
|
|
21
21
|
* disk and load on demand; cheap to update (no data duplication) and re-compute
|
|
22
|
-
* the `stateRoot`.
|
|
22
|
+
* the `stateRoot`. Used in LMDB.
|
|
23
23
|
* - `SerializedState<StateEntries>`: serialized state represented as a simple in-memory
|
|
24
24
|
* hashmap of `key -> value` entries.
|
|
25
25
|
*/
|
|
@@ -8,9 +8,40 @@ import type { WorkerConfig } from "#@typeberry/workers-api";
|
|
|
8
8
|
import { ThreadPort, type TransferablePort } from "./port.js";
|
|
9
9
|
export { FjallValuesSession };
|
|
10
10
|
/** Persistent regular-node backend. */
|
|
11
|
-
export type PersistentBackend = "fjall";
|
|
11
|
+
export type PersistentBackend = "lmdb" | "fjall";
|
|
12
12
|
/** Transferable worker config for persistent regular-node workers. */
|
|
13
|
-
export type PersistentWorkerConfig<T> = FjallWorkerConfig<T>;
|
|
13
|
+
export type PersistentWorkerConfig<T> = LmdbWorkerConfig<T> | FjallWorkerConfig<T>;
|
|
14
|
+
/**
|
|
15
|
+
* Worker config for node.js, backed by the LMDB database.
|
|
16
|
+
*
|
|
17
|
+
* @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
|
|
18
|
+
*/
|
|
19
|
+
export declare class LmdbWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
|
|
20
|
+
readonly nodeName: string;
|
|
21
|
+
readonly chainSpec: ChainSpec;
|
|
22
|
+
readonly workerParams: T;
|
|
23
|
+
readonly dbPath: string;
|
|
24
|
+
readonly blake2b: Blake2b;
|
|
25
|
+
readonly ports: Map<string, ThreadPort>;
|
|
26
|
+
readonly ephemeral: boolean;
|
|
27
|
+
static new<T>({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, }: {
|
|
28
|
+
nodeName: string;
|
|
29
|
+
chainSpec: ChainSpec;
|
|
30
|
+
workerParams: T;
|
|
31
|
+
dbPath: string;
|
|
32
|
+
blake2b: Blake2b;
|
|
33
|
+
ports?: Map<string, ThreadPort>;
|
|
34
|
+
ephemeral?: boolean;
|
|
35
|
+
}): LmdbWorkerConfig<T>;
|
|
36
|
+
/** Restore node config from a transferable config object. */
|
|
37
|
+
static fromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<LmdbWorkerConfig<T>>;
|
|
38
|
+
private constructor();
|
|
39
|
+
openDatabase(options?: {
|
|
40
|
+
readonly: boolean;
|
|
41
|
+
}): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
|
|
42
|
+
/** Convert this config into a thread-transferable object. */
|
|
43
|
+
intoTransferable(paramsCodec: Encode<T>): TransferableConfig;
|
|
44
|
+
}
|
|
14
45
|
/** Worker config for node.js, backed by a shared fjall engine. */
|
|
15
46
|
export declare class FjallWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
|
|
16
47
|
readonly nodeName: string;
|
|
@@ -84,10 +115,11 @@ export declare class InMemWorkerConfig<T = undefined> implements WorkerConfig<T,
|
|
|
84
115
|
}): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
|
|
85
116
|
}
|
|
86
117
|
/** Persistent values store backing the hybrid config. */
|
|
87
|
-
export type HybridBackend = "fjall";
|
|
118
|
+
export type HybridBackend = "lmdb" | "fjall";
|
|
88
119
|
/**
|
|
89
120
|
* Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
|
|
90
|
-
* but large values persisted to disk
|
|
121
|
+
* but large values persisted to disk. The `backend` picks where the values go
|
|
122
|
+
* (lmdb or fjall).
|
|
91
123
|
*
|
|
92
124
|
* fjall opens its keyspace asynchronously, that is why `new` here is async.
|
|
93
125
|
*
|
|
@@ -109,7 +141,7 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
|
|
|
109
141
|
readonly ephemeral: boolean;
|
|
110
142
|
readonly compression: boolean;
|
|
111
143
|
private readonly states;
|
|
112
|
-
static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, sharedFjallSession, }: {
|
|
144
|
+
static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, backend, sharedFjallSession, }: {
|
|
113
145
|
nodeName: string;
|
|
114
146
|
chainSpec: ChainSpec;
|
|
115
147
|
workerParams: T;
|
|
@@ -117,10 +149,12 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
|
|
|
117
149
|
dbPath: string;
|
|
118
150
|
ephemeral?: boolean;
|
|
119
151
|
compression?: boolean;
|
|
152
|
+
backend?: HybridBackend;
|
|
120
153
|
/**
|
|
121
154
|
* Reuse an already-open fjall values session instead of opening a fresh
|
|
122
155
|
* keyspace. The fuzz target opens one per run and passes it on every reset,
|
|
123
|
-
* so only the in-memory blocks/leaf sets are rebuilt per vector.
|
|
156
|
+
* so only the in-memory blocks/leaf sets are rebuilt per vector. Ignored
|
|
157
|
+
* unless `backend === "fjall"`.
|
|
124
158
|
*/
|
|
125
159
|
sharedFjallSession?: FjallValuesSession;
|
|
126
160
|
}): Promise<HybridWorkerConfig<T>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAKL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAKL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAOnC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAI9D,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,uCAAuC;AACvC,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,OAAO,CAAC;AAEjD,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAEnF;;;;GAIG;AACH,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAI9B,SAAS,EAAE,OAAO;IA/CpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAgBpF,OAAO;IAaD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAahD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAU7D;AAED,kEAAkE;AAClE,qBAAa,iBAAiB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0C7E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAE9B,SAAS,EAAE,OAAO;aAClB,cAAc,EAAE,MAAM,GAAG,SAAS;IAjDpD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,EACjB,cAAc,GACf,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAiBpF,OAAO;IAYD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAuChD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAW7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,EAAE,iBAAiB,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAgBF,qEAAqE;AACrE,wBAAsB,gCAAgC,CAAC,CAAC,EACtD,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EACvB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAOpC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CAQjD;AAED,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;AAE7C;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0CnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;aAClB,WAAW,EAAE,OAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;WAhDZ,GAAG,CAAC,CAAC,EAAE,EAClB,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,EACjB,WAAkB,EAClB,OAAgB,EAChB,kBAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,OAAO,CAAC,EAAE,aAAa,CAAC;QACxB;;;;;WAKG;QACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KACzC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAYlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IAExC,OAAO;IAaD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CASjD"}
|