@typeberry/lib 0.10.0-9e3af22 → 0.11.0-411c828
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -3
- package/packages/jam/config-node/node-config.d.ts +1 -11
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +3 -25
- package/packages/jam/config-node/node-config.test.js +3 -7
- package/packages/jam/database/states.d.ts +1 -8
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database/value-refs.d.ts +4 -5
- package/packages/jam/database/value-refs.d.ts.map +1 -1
- package/packages/jam/database-fjall/blocks.d.ts +1 -0
- package/packages/jam/database-fjall/blocks.d.ts.map +1 -1
- package/packages/jam/database-fjall/blocks.js +8 -3
- package/packages/jam/database-fjall/hybrid-states.d.ts +10 -4
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.js +22 -6
- package/packages/jam/database-fjall/root.d.ts +2 -0
- package/packages/jam/database-fjall/root.d.ts.map +1 -1
- package/packages/jam/database-fjall/root.js +8 -1
- package/packages/jam/database-fjall/root.test.d.ts +2 -0
- package/packages/jam/database-fjall/root.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/root.test.js +23 -0
- package/packages/jam/database-fjall/states.d.ts +1 -0
- package/packages/jam/database-fjall/states.d.ts.map +1 -1
- package/packages/jam/database-fjall/states.js +6 -3
- package/packages/jam/node/export.d.ts.map +1 -1
- package/packages/jam/node/export.js +8 -17
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +51 -46
- package/packages/jam/node/main-importer.d.ts +6 -11
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +14 -28
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +6 -16
- package/packages/jam/rpc/test/e2e-server.d.ts.map +1 -1
- package/packages/jam/rpc/test/e2e-server.js +3 -5
- package/packages/jam/rpc/test/e2e.js +1 -1
- package/packages/jam/state-merkleization/index.d.ts +1 -1
- package/packages/jam/state-merkleization/index.js +1 -1
- package/packages/workers/api-node/config.d.ts +14 -46
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +31 -96
- package/packages/workers/api-node/config.test.js +82 -63
- package/packages/jam/database-lmdb/blocks.d.ts +0 -24
- package/packages/jam/database-lmdb/blocks.d.ts.map +0 -1
- package/packages/jam/database-lmdb/blocks.js +0 -82
- package/packages/jam/database-lmdb/hybrid-states.d.ts +0 -56
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +0 -1
- package/packages/jam/database-lmdb/hybrid-states.js +0 -149
- package/packages/jam/database-lmdb/hybrid-states.test.d.ts +0 -2
- package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map +0 -1
- package/packages/jam/database-lmdb/hybrid-states.test.js +0 -180
- package/packages/jam/database-lmdb/index.d.ts +0 -5
- package/packages/jam/database-lmdb/index.d.ts.map +0 -1
- package/packages/jam/database-lmdb/index.js +0 -4
- package/packages/jam/database-lmdb/root.d.ts +0 -26
- package/packages/jam/database-lmdb/root.d.ts.map +0 -1
- package/packages/jam/database-lmdb/root.js +0 -53
- package/packages/jam/database-lmdb/states.d.ts +0 -76
- package/packages/jam/database-lmdb/states.d.ts.map +0 -1
- package/packages/jam/database-lmdb/states.js +0 -145
- package/packages/jam/database-lmdb/states.test.d.ts +0 -2
- package/packages/jam/database-lmdb/states.test.d.ts.map +0 -1
- package/packages/jam/database-lmdb/states.test.js +0 -202
|
@@ -2,79 +2,11 @@ import { Decoder, Encoder } from "#@typeberry/codec";
|
|
|
2
2
|
import { ChainSpec } from "#@typeberry/config";
|
|
3
3
|
import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
|
|
4
4
|
import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
|
|
5
|
-
import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
|
|
6
5
|
import { Blake2b } from "#@typeberry/hash";
|
|
7
6
|
import { ThreadPort } from "./port.js";
|
|
8
|
-
// Re-exported so the fuzz target can open one
|
|
9
|
-
//
|
|
10
|
-
export { FjallValuesSession };
|
|
11
|
-
/**
|
|
12
|
-
* Worker config for node.js, backed by the LMDB database.
|
|
13
|
-
*
|
|
14
|
-
* @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
|
|
15
|
-
*/
|
|
16
|
-
export class LmdbWorkerConfig {
|
|
17
|
-
nodeName;
|
|
18
|
-
chainSpec;
|
|
19
|
-
workerParams;
|
|
20
|
-
dbPath;
|
|
21
|
-
blake2b;
|
|
22
|
-
ports;
|
|
23
|
-
ephemeral;
|
|
24
|
-
static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, }) {
|
|
25
|
-
return new LmdbWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral);
|
|
26
|
-
}
|
|
27
|
-
/** Restore node config from a transferable config object. */
|
|
28
|
-
static async fromTransferable(decodeParams, config) {
|
|
29
|
-
if (config.databaseBackend !== "lmdb") {
|
|
30
|
-
throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
|
|
31
|
-
}
|
|
32
|
-
const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
|
|
33
|
-
return LmdbWorkerConfig.new({
|
|
34
|
-
nodeName: config.nodeName,
|
|
35
|
-
chainSpec,
|
|
36
|
-
workerParams,
|
|
37
|
-
dbPath: config.dbPath,
|
|
38
|
-
blake2b,
|
|
39
|
-
ports,
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
43
|
-
// When set, the underlying database skips fsync. Only safe for throwaway
|
|
44
|
-
// databases (the fuzz target wipes on reset). Not transferred to worker
|
|
45
|
-
// threads, so the durable main node path always gets the default.
|
|
46
|
-
ephemeral = false) {
|
|
47
|
-
this.nodeName = nodeName;
|
|
48
|
-
this.chainSpec = chainSpec;
|
|
49
|
-
this.workerParams = workerParams;
|
|
50
|
-
this.dbPath = dbPath;
|
|
51
|
-
this.blake2b = blake2b;
|
|
52
|
-
this.ports = ports;
|
|
53
|
-
this.ephemeral = ephemeral;
|
|
54
|
-
}
|
|
55
|
-
async openDatabase(options = { readonly: true }) {
|
|
56
|
-
const lmdb = LmdbRoot.new(this.dbPath, {
|
|
57
|
-
readOnly: options.readonly,
|
|
58
|
-
ephemeral: this.ephemeral,
|
|
59
|
-
});
|
|
60
|
-
return {
|
|
61
|
-
getBlocksDb: () => LmdbBlocks.new(this.chainSpec, lmdb),
|
|
62
|
-
getStatesDb: () => LmdbStates.new(this.chainSpec, this.blake2b, lmdb),
|
|
63
|
-
close: async () => await lmdb.close(),
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
/** Convert this config into a thread-transferable object. */
|
|
67
|
-
intoTransferable(paramsCodec) {
|
|
68
|
-
return {
|
|
69
|
-
databaseBackend: "lmdb",
|
|
70
|
-
nodeName: this.nodeName,
|
|
71
|
-
chainSpec: this.chainSpec,
|
|
72
|
-
workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
|
|
73
|
-
dbPath: this.dbPath,
|
|
74
|
-
workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
}
|
|
7
|
+
// Re-exported so the fuzz target can open one keyspace per run and reuse it
|
|
8
|
+
// across resets.
|
|
9
|
+
export { FjallRoot, FjallValuesSession };
|
|
78
10
|
/** Worker config for node.js, backed by a shared fjall engine. */
|
|
79
11
|
export class FjallWorkerConfig {
|
|
80
12
|
nodeName;
|
|
@@ -85,8 +17,9 @@ export class FjallWorkerConfig {
|
|
|
85
17
|
ports;
|
|
86
18
|
ephemeral;
|
|
87
19
|
cacheSizeBytes;
|
|
88
|
-
|
|
89
|
-
|
|
20
|
+
sharedFjallKeyspace;
|
|
21
|
+
static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, sharedFjallKeyspace, }) {
|
|
22
|
+
return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes, sharedFjallKeyspace);
|
|
90
23
|
}
|
|
91
24
|
/** Restore node config from a transferable config object. */
|
|
92
25
|
static async fromTransferable(decodeParams, config) {
|
|
@@ -106,7 +39,7 @@ export class FjallWorkerConfig {
|
|
|
106
39
|
}
|
|
107
40
|
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
108
41
|
// Kept for the fuzz/importer path. When set, persist() is skipped.
|
|
109
|
-
ephemeral = false, cacheSizeBytes = undefined) {
|
|
42
|
+
ephemeral = false, cacheSizeBytes = undefined, sharedFjallKeyspace = undefined) {
|
|
110
43
|
this.nodeName = nodeName;
|
|
111
44
|
this.chainSpec = chainSpec;
|
|
112
45
|
this.workerParams = workerParams;
|
|
@@ -115,13 +48,19 @@ export class FjallWorkerConfig {
|
|
|
115
48
|
this.ports = ports;
|
|
116
49
|
this.ephemeral = ephemeral;
|
|
117
50
|
this.cacheSizeBytes = cacheSizeBytes;
|
|
51
|
+
this.sharedFjallKeyspace = sharedFjallKeyspace;
|
|
118
52
|
}
|
|
119
53
|
async openDatabase(options = { readonly: true }) {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
54
|
+
if (this.sharedFjallKeyspace !== undefined && options.readonly) {
|
|
55
|
+
throw new Error("Cannot open a read-only fjall database from a shared writable keyspace.");
|
|
56
|
+
}
|
|
57
|
+
const fjall = this.sharedFjallKeyspace ??
|
|
58
|
+
(await FjallRoot.open(this.dbPath, {
|
|
59
|
+
readOnly: options.readonly,
|
|
60
|
+
ephemeral: this.ephemeral,
|
|
61
|
+
cacheSizeBytes: this.cacheSizeBytes,
|
|
62
|
+
}));
|
|
63
|
+
const ownsFjall = this.sharedFjallKeyspace === undefined;
|
|
125
64
|
let blocks = null;
|
|
126
65
|
let states = null;
|
|
127
66
|
try {
|
|
@@ -131,7 +70,9 @@ export class FjallWorkerConfig {
|
|
|
131
70
|
]);
|
|
132
71
|
}
|
|
133
72
|
catch (e) {
|
|
134
|
-
|
|
73
|
+
if (ownsFjall) {
|
|
74
|
+
await fjall.close();
|
|
75
|
+
}
|
|
135
76
|
throw e;
|
|
136
77
|
}
|
|
137
78
|
return {
|
|
@@ -150,7 +91,9 @@ export class FjallWorkerConfig {
|
|
|
150
91
|
close: async () => {
|
|
151
92
|
blocks = null;
|
|
152
93
|
states = null;
|
|
153
|
-
|
|
94
|
+
if (ownsFjall) {
|
|
95
|
+
await fjall.close();
|
|
96
|
+
}
|
|
154
97
|
},
|
|
155
98
|
};
|
|
156
99
|
}
|
|
@@ -181,12 +124,7 @@ async function decodeTransferableConfig(decodeParams, config) {
|
|
|
181
124
|
}
|
|
182
125
|
/** Restore a persistent worker config from its transferable form. */
|
|
183
126
|
export async function persistentConfigFromTransferable(decodeParams, config) {
|
|
184
|
-
|
|
185
|
-
case "lmdb":
|
|
186
|
-
return LmdbWorkerConfig.fromTransferable(decodeParams, config);
|
|
187
|
-
case "fjall":
|
|
188
|
-
return FjallWorkerConfig.fromTransferable(decodeParams, config);
|
|
189
|
-
}
|
|
127
|
+
return FjallWorkerConfig.fromTransferable(decodeParams, config);
|
|
190
128
|
}
|
|
191
129
|
/**
|
|
192
130
|
* Collect the transferable objects (communication ports) embedded in a config.
|
|
@@ -232,8 +170,7 @@ export class InMemWorkerConfig {
|
|
|
232
170
|
}
|
|
233
171
|
/**
|
|
234
172
|
* Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
|
|
235
|
-
* but large values persisted to disk
|
|
236
|
-
* (lmdb or fjall).
|
|
173
|
+
* but large values persisted to disk via fjall.
|
|
237
174
|
*
|
|
238
175
|
* fjall opens its keyspace asynchronously, that is why `new` here is async.
|
|
239
176
|
*
|
|
@@ -255,14 +192,12 @@ export class HybridWorkerConfig {
|
|
|
255
192
|
ephemeral;
|
|
256
193
|
compression;
|
|
257
194
|
states;
|
|
258
|
-
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true,
|
|
195
|
+
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, sharedFjallKeyspace, }) {
|
|
259
196
|
// The values store is created once here and shared across reopen. When a
|
|
260
|
-
//
|
|
261
|
-
const states =
|
|
262
|
-
?
|
|
263
|
-
|
|
264
|
-
: await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
|
|
265
|
-
: LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
|
|
197
|
+
// keyspace is given (fuzz reset reuse) we wrap it instead of opening a new one.
|
|
198
|
+
const states = sharedFjallKeyspace !== undefined
|
|
199
|
+
? await FjallHybridSerializedStates.fromRoot(chainSpec, blake2b, sharedFjallKeyspace)
|
|
200
|
+
: await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral });
|
|
266
201
|
return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
|
|
267
202
|
}
|
|
268
203
|
blocks;
|
|
@@ -1,46 +1,15 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
|
-
import { MessageChannel } from "node:worker_threads";
|
|
5
4
|
import { Bytes } from "#@typeberry/bytes";
|
|
6
5
|
import { codec } from "#@typeberry/codec";
|
|
7
6
|
import { tinyChainSpec } from "#@typeberry/config";
|
|
7
|
+
import { FjallRoot } from "#@typeberry/database-fjall";
|
|
8
8
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
9
9
|
import { tryAsU32 } from "#@typeberry/numbers";
|
|
10
|
-
import { configTransferList, FjallWorkerConfig, HybridWorkerConfig
|
|
10
|
+
import { configTransferList, FjallWorkerConfig, HybridWorkerConfig } from "./config.js";
|
|
11
11
|
import { ThreadPort } from "./port.js";
|
|
12
12
|
const spec = tinyChainSpec;
|
|
13
|
-
describe("LmdbWorkerConfig transfer list", () => {
|
|
14
|
-
it("surfaces embedded worker ports so they can be transferred", async () => {
|
|
15
|
-
const blake2b = await Blake2b.createHasher();
|
|
16
|
-
const [portA, portB] = ThreadPort.pair(spec);
|
|
17
|
-
const config = LmdbWorkerConfig.new({
|
|
18
|
-
nodeName: "node",
|
|
19
|
-
chainSpec: spec,
|
|
20
|
-
workerParams: tryAsU32(7),
|
|
21
|
-
dbPath: "db",
|
|
22
|
-
blake2b,
|
|
23
|
-
ports: new Map([["authorship-network", portA]]),
|
|
24
|
-
});
|
|
25
|
-
const transferable = config.intoTransferable(codec.varU32);
|
|
26
|
-
const transferList = configTransferList(transferable);
|
|
27
|
-
// the single embedded comms port must be reported for transfer
|
|
28
|
-
assert.strictEqual(transferList.length, 1);
|
|
29
|
-
const sink = new MessageChannel();
|
|
30
|
-
try {
|
|
31
|
-
// reproduces the bug: a config carrying a port cannot be cloned without
|
|
32
|
-
// listing that port in the transfer list.
|
|
33
|
-
assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
|
|
34
|
-
// with the ports surfaced, posting succeeds.
|
|
35
|
-
assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
|
|
36
|
-
}
|
|
37
|
-
finally {
|
|
38
|
-
sink.port1.close();
|
|
39
|
-
sink.port2.close();
|
|
40
|
-
portB.close();
|
|
41
|
-
}
|
|
42
|
-
});
|
|
43
|
-
});
|
|
44
13
|
describe("FjallWorkerConfig transfer list", () => {
|
|
45
14
|
it("surfaces embedded worker ports and marks the backend", async () => {
|
|
46
15
|
const blake2b = await Blake2b.createHasher();
|
|
@@ -88,40 +57,90 @@ describe("FjallWorkerConfig transfer list", () => {
|
|
|
88
57
|
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
89
58
|
}
|
|
90
59
|
});
|
|
60
|
+
it("can borrow a shared fjall root without closing it", async () => {
|
|
61
|
+
const blake2b = await Blake2b.createHasher();
|
|
62
|
+
const dbPath = fs.mkdtempSync("typeberry-fjall-worker-shared-");
|
|
63
|
+
const root = await FjallRoot.open(dbPath, { ephemeral: true });
|
|
64
|
+
const config = FjallWorkerConfig.new({
|
|
65
|
+
nodeName: "node",
|
|
66
|
+
chainSpec: spec,
|
|
67
|
+
workerParams: undefined,
|
|
68
|
+
dbPath,
|
|
69
|
+
blake2b,
|
|
70
|
+
sharedFjallKeyspace: root,
|
|
71
|
+
});
|
|
72
|
+
try {
|
|
73
|
+
const db = await config.openDatabase({ readonly: false });
|
|
74
|
+
await db.close();
|
|
75
|
+
await root.deletePartition("headers");
|
|
76
|
+
const headers = await root.writablePartition("headers");
|
|
77
|
+
assert.strictEqual(headers.get(Bytes.zero(HASH_SIZE).raw), null);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await root.close();
|
|
81
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
82
|
+
}
|
|
83
|
+
});
|
|
91
84
|
});
|
|
92
85
|
describe("HybridWorkerConfig", () => {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
86
|
+
it("constructs and opens a fjall-backed hybrid db", async () => {
|
|
87
|
+
const blake2b = await Blake2b.createHasher();
|
|
88
|
+
const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-");
|
|
89
|
+
try {
|
|
90
|
+
const config = await HybridWorkerConfig.new({
|
|
91
|
+
nodeName: "node",
|
|
92
|
+
chainSpec: spec,
|
|
93
|
+
workerParams: undefined,
|
|
94
|
+
blake2b,
|
|
95
|
+
dbPath,
|
|
96
|
+
ephemeral: true,
|
|
97
|
+
});
|
|
98
|
+
const db = await config.openDatabase({ readonly: false });
|
|
99
|
+
const states = db.getStatesDb();
|
|
99
100
|
try {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
chainSpec: spec,
|
|
103
|
-
workerParams: undefined,
|
|
104
|
-
blake2b,
|
|
105
|
-
dbPath,
|
|
106
|
-
ephemeral: true,
|
|
107
|
-
backend,
|
|
108
|
-
});
|
|
109
|
-
const db = await config.openDatabase({ readonly: false });
|
|
110
|
-
const states = db.getStatesDb();
|
|
111
|
-
try {
|
|
112
|
-
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
113
|
-
assert.notStrictEqual(states, undefined);
|
|
114
|
-
}
|
|
115
|
-
finally {
|
|
116
|
-
// The values store owns the on-disk resources (the no-op db.close()
|
|
117
|
-
// does not), so close it explicitly to release the fjall keyspace.
|
|
118
|
-
await states.close();
|
|
119
|
-
await db.close();
|
|
120
|
-
}
|
|
101
|
+
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
102
|
+
assert.notStrictEqual(states, undefined);
|
|
121
103
|
}
|
|
122
104
|
finally {
|
|
123
|
-
|
|
105
|
+
// The values store owns the on-disk resources (the no-op db.close()
|
|
106
|
+
// does not), so close it explicitly to release the fjall keyspace.
|
|
107
|
+
await states.close();
|
|
108
|
+
await db.close();
|
|
124
109
|
}
|
|
125
|
-
}
|
|
126
|
-
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
it("can borrow a shared fjall root for the values store", async () => {
|
|
116
|
+
const blake2b = await Blake2b.createHasher();
|
|
117
|
+
const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-shared-");
|
|
118
|
+
const root = await FjallRoot.open(dbPath, { ephemeral: true });
|
|
119
|
+
try {
|
|
120
|
+
const config = await HybridWorkerConfig.new({
|
|
121
|
+
nodeName: "node",
|
|
122
|
+
chainSpec: spec,
|
|
123
|
+
workerParams: undefined,
|
|
124
|
+
blake2b,
|
|
125
|
+
dbPath,
|
|
126
|
+
ephemeral: true,
|
|
127
|
+
sharedFjallKeyspace: root,
|
|
128
|
+
});
|
|
129
|
+
const db = await config.openDatabase({ readonly: false });
|
|
130
|
+
const states = db.getStatesDb();
|
|
131
|
+
try {
|
|
132
|
+
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
133
|
+
assert.notStrictEqual(states, undefined);
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
await states.close();
|
|
137
|
+
await db.close();
|
|
138
|
+
}
|
|
139
|
+
await root.deletePartition("values");
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
await root.close();
|
|
143
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
144
|
+
}
|
|
145
|
+
});
|
|
127
146
|
});
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { type BlockView, type ExtrinsicView, type HeaderHash, type HeaderView, type StateRootHash } from "#@typeberry/block";
|
|
2
|
-
import type { ChainSpec } from "#@typeberry/config";
|
|
3
|
-
import type { BlocksDb } from "#@typeberry/database/blocks.js";
|
|
4
|
-
import { type WithHash } from "#@typeberry/hash";
|
|
5
|
-
import type { LmdbRoot } from "./root.js";
|
|
6
|
-
export declare class LmdbBlocks implements BlocksDb {
|
|
7
|
-
private readonly chainSpec;
|
|
8
|
-
private readonly root;
|
|
9
|
-
private readonly extrinsics;
|
|
10
|
-
private readonly headers;
|
|
11
|
-
private readonly postStateRoots;
|
|
12
|
-
static new(chainSpec: ChainSpec, root: LmdbRoot): LmdbBlocks;
|
|
13
|
-
private constructor();
|
|
14
|
-
setPostStateRoot(hash: HeaderHash, postStateRoot: StateRootHash): Promise<void>;
|
|
15
|
-
getPostStateRoot(hash: HeaderHash): StateRootHash | null;
|
|
16
|
-
insertBlock(block: WithHash<HeaderHash, BlockView>): Promise<void>;
|
|
17
|
-
setBestHeaderHash(hash: HeaderHash): Promise<void>;
|
|
18
|
-
getBestHeaderHash(): HeaderHash;
|
|
19
|
-
getHeader(hash: HeaderHash): HeaderView | null;
|
|
20
|
-
getExtrinsic(hash: HeaderHash): ExtrinsicView | null;
|
|
21
|
-
markUnused(hash: HeaderHash): void;
|
|
22
|
-
close(): Promise<void>;
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=blocks.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAQjD,qBAAa,UAAW,YAAW,QAAQ;IAUvC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAEvC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ;IAI/C,OAAO;IAeD,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrF,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQlD,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IASlE,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxD,iBAAiB,IAAI,UAAU;IAS/B,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAS9C,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQpD,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAM5B,KAAK;CAGZ"}
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { Extrinsic, Header, } from "#@typeberry/block";
|
|
2
|
-
import { Bytes } from "#@typeberry/bytes";
|
|
3
|
-
import { Decoder } from "#@typeberry/codec";
|
|
4
|
-
import { HASH_SIZE } from "#@typeberry/hash";
|
|
5
|
-
const BEST_BLOCK = "best hash and posterior state root";
|
|
6
|
-
// TODO [ToDr] consider having a changeset for transactions,
|
|
7
|
-
// where we store all `insert ++ key ++ value` and `remove ++ key`
|
|
8
|
-
// in a single `Uint8Array` JAM-encoded. That could then
|
|
9
|
-
// be efficiently transferred between threads.
|
|
10
|
-
export class LmdbBlocks {
|
|
11
|
-
chainSpec;
|
|
12
|
-
root;
|
|
13
|
-
extrinsics;
|
|
14
|
-
headers;
|
|
15
|
-
postStateRoots;
|
|
16
|
-
static new(chainSpec, root) {
|
|
17
|
-
return new LmdbBlocks(chainSpec, root);
|
|
18
|
-
}
|
|
19
|
-
constructor(chainSpec, root) {
|
|
20
|
-
this.chainSpec = chainSpec;
|
|
21
|
-
this.root = root;
|
|
22
|
-
// NOTE [ToDr] Extrinsics are stored under header hash, not their hash. Revise if it's an issue.
|
|
23
|
-
this.extrinsics = this.root.subDb("extrinsics");
|
|
24
|
-
this.headers = this.root.subDb("headers");
|
|
25
|
-
// NOTE [ToDr] We currently store all posterior state roots, however it's
|
|
26
|
-
// most likely very redundant. We probably only need to store the posterior
|
|
27
|
-
// state roots of recent blocks to be able to quickly resolve forks
|
|
28
|
-
// OR we need a way to be able to traverse the blocks history forward
|
|
29
|
-
// (i.e. know what next block(s) is).
|
|
30
|
-
this.postStateRoots = this.root.subDb("postStateRoots");
|
|
31
|
-
}
|
|
32
|
-
async setPostStateRoot(hash, postStateRoot) {
|
|
33
|
-
await this.postStateRoots.put(hash.raw, postStateRoot.raw);
|
|
34
|
-
}
|
|
35
|
-
getPostStateRoot(hash) {
|
|
36
|
-
const postStateRoot = this.postStateRoots.get(hash.raw);
|
|
37
|
-
if (postStateRoot === undefined) {
|
|
38
|
-
return null;
|
|
39
|
-
}
|
|
40
|
-
return Bytes.fromBlob(postStateRoot, HASH_SIZE).asOpaque();
|
|
41
|
-
}
|
|
42
|
-
async insertBlock(block) {
|
|
43
|
-
const header = block.data.header.view().encoded();
|
|
44
|
-
const extrinsic = block.data.extrinsic.view().encoded();
|
|
45
|
-
await this.root.db.transaction(() => {
|
|
46
|
-
this.headers.put(block.hash.raw, header.raw);
|
|
47
|
-
this.extrinsics.put(block.hash.raw, extrinsic.raw);
|
|
48
|
-
});
|
|
49
|
-
}
|
|
50
|
-
async setBestHeaderHash(hash) {
|
|
51
|
-
await this.root.db.put(BEST_BLOCK, hash.raw);
|
|
52
|
-
}
|
|
53
|
-
getBestHeaderHash() {
|
|
54
|
-
const bestHeaderHash = this.root.db.get(BEST_BLOCK);
|
|
55
|
-
if (bestHeaderHash === undefined) {
|
|
56
|
-
return Bytes.zero(HASH_SIZE).asOpaque();
|
|
57
|
-
}
|
|
58
|
-
return Bytes.fromBlob(bestHeaderHash, HASH_SIZE).asOpaque();
|
|
59
|
-
}
|
|
60
|
-
getHeader(hash) {
|
|
61
|
-
const data = this.headers.get(hash.raw);
|
|
62
|
-
if (data === undefined) {
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
return Decoder.decodeObject(Header.Codec.View, data, this.chainSpec);
|
|
66
|
-
}
|
|
67
|
-
getExtrinsic(hash) {
|
|
68
|
-
const data = this.extrinsics.get(hash.raw);
|
|
69
|
-
if (data === undefined) {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
return Decoder.decodeObject(Extrinsic.Codec.View, data, this.chainSpec);
|
|
73
|
-
}
|
|
74
|
-
markUnused(hash) {
|
|
75
|
-
this.headers.removeSync(hash.raw);
|
|
76
|
-
this.extrinsics.removeSync(hash.raw);
|
|
77
|
-
this.postStateRoots.removeSync(hash.raw);
|
|
78
|
-
}
|
|
79
|
-
async close() {
|
|
80
|
-
await Promise.all([this.headers.close(), this.extrinsics.close(), this.postStateRoots.close()]);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import type { HeaderHash, StateRootHash } from "#@typeberry/block";
|
|
2
|
-
import type { ChainSpec } from "#@typeberry/config";
|
|
3
|
-
import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
|
|
4
|
-
import type { Blake2b } from "#@typeberry/hash";
|
|
5
|
-
import type { ServicesUpdate, State } from "#@typeberry/state";
|
|
6
|
-
import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
|
|
7
|
-
import { OK, Result } from "#@typeberry/utils";
|
|
8
|
-
/**
|
|
9
|
-
* Hybrid serialized-states db.
|
|
10
|
-
*
|
|
11
|
-
* States (leafs) are kept in-memory, but large values are persisted to lmdb.
|
|
12
|
-
* Reads go straight to lmdb, which keeps its own page cache.
|
|
13
|
-
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
14
|
-
* keep the heap usage bounded.
|
|
15
|
-
*
|
|
16
|
-
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
17
|
-
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
18
|
-
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
19
|
-
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
20
|
-
* run are never collected.
|
|
21
|
-
*/
|
|
22
|
-
export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
|
|
23
|
-
private readonly spec;
|
|
24
|
-
private readonly blake2b;
|
|
25
|
-
private readonly root;
|
|
26
|
-
private readonly inMemStates;
|
|
27
|
-
private readonly lmdbValues;
|
|
28
|
-
private readonly valuesDb;
|
|
29
|
-
private readonly refsStore;
|
|
30
|
-
private readonly refs;
|
|
31
|
-
private pendingCleanup;
|
|
32
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
|
|
33
|
-
spec: ChainSpec;
|
|
34
|
-
blake2b: Blake2b;
|
|
35
|
-
dbPath: string;
|
|
36
|
-
readOnly?: boolean;
|
|
37
|
-
ephemeral?: boolean;
|
|
38
|
-
compression?: boolean;
|
|
39
|
-
}): HybridSerializedStates;
|
|
40
|
-
private constructor();
|
|
41
|
-
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
42
|
-
updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
43
|
-
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
44
|
-
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
45
|
-
commitFinalized(headers: HeaderHash[]): void;
|
|
46
|
-
markUnused(header: HeaderHash): void;
|
|
47
|
-
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
48
|
-
private applyRefs;
|
|
49
|
-
diskSizeInBytes(): number | null;
|
|
50
|
-
close(): Promise<void>;
|
|
51
|
-
/** Write new large values to LMDB in one transaction. */
|
|
52
|
-
private writeValues;
|
|
53
|
-
/** Read a value from LMDB. */
|
|
54
|
-
private readValue;
|
|
55
|
-
}
|
|
56
|
-
//# sourceMappingURL=hybrid-states.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
|