@typeberry/lib 0.8.4-faebc7a → 0.9.0-1c3f157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/configs/index.d.ts +30 -1
- package/packages/configs/index.d.ts.map +1 -1
- package/packages/configs/index.js +4 -2
- package/packages/configs/typeberry-dev-full.json +29 -0
- package/packages/core/bytes/bytes.d.ts +1 -0
- package/packages/core/bytes/bytes.d.ts.map +1 -1
- package/packages/core/bytes/bytes.js +8 -0
- package/packages/core/collections/sorted-array.d.ts +1 -1
- package/packages/core/collections/sorted-array.d.ts.map +1 -1
- package/packages/core/collections/sorted-array.js +4 -2
- package/packages/core/collections/sorted-array.test.js +10 -0
- package/packages/core/collections/sorted-set.d.ts +1 -1
- package/packages/core/collections/sorted-set.d.ts.map +1 -1
- package/packages/core/collections/sorted-set.js +6 -2
- package/packages/core/collections/sorted-set.test.js +24 -0
- package/packages/core/utils/debug.d.ts +4 -2
- package/packages/core/utils/debug.d.ts.map +1 -1
- package/packages/core/utils/debug.js +18 -13
- package/packages/core/utils/debug.test.js +12 -6
- package/packages/core/utils/index.d.ts +1 -0
- package/packages/core/utils/index.d.ts.map +1 -1
- package/packages/core/utils/index.js +1 -0
- package/packages/core/utils/shutdown.d.ts +18 -0
- package/packages/core/utils/shutdown.d.ts.map +1 -0
- package/packages/core/utils/shutdown.js +3 -0
- package/packages/core/utils/shutdown.node.d.ts +5 -0
- package/packages/core/utils/shutdown.node.d.ts.map +1 -0
- package/packages/core/utils/shutdown.node.js +76 -0
- package/packages/core/utils/shutdown.test.d.ts +2 -0
- package/packages/core/utils/shutdown.test.d.ts.map +1 -0
- package/packages/core/utils/shutdown.test.js +164 -0
- package/packages/extensions/ipc/server.d.ts.map +1 -1
- package/packages/extensions/ipc/server.js +7 -0
- package/packages/jam/config-node/node-config.d.ts +22 -3
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +33 -6
- package/packages/jam/config-node/node-config.test.js +11 -4
- package/packages/jam/database/index.d.ts +1 -0
- package/packages/jam/database/index.d.ts.map +1 -1
- package/packages/jam/database/index.js +1 -0
- package/packages/jam/database/leaf-db-update.d.ts +1 -0
- package/packages/jam/database/leaf-db-update.d.ts.map +1 -1
- package/packages/jam/database/leaf-db-update.js +22 -5
- package/packages/jam/database/leaf-db-update.test.d.ts +2 -0
- package/packages/jam/database/leaf-db-update.test.d.ts.map +1 -0
- package/packages/jam/database/leaf-db-update.test.js +81 -0
- package/packages/jam/database/serialized-states-db.d.ts +1 -0
- package/packages/jam/database/serialized-states-db.d.ts.map +1 -1
- package/packages/jam/database/serialized-states-db.js +4 -0
- package/packages/jam/database/states.d.ts +11 -0
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database/states.js +3 -0
- package/packages/jam/database/value-refs.d.ts +117 -0
- package/packages/jam/database/value-refs.d.ts.map +1 -0
- package/packages/jam/database/value-refs.js +206 -0
- package/packages/jam/database/value-refs.test.d.ts +2 -0
- package/packages/jam/database/value-refs.test.d.ts.map +1 -0
- package/packages/jam/database/value-refs.test.js +165 -0
- package/packages/jam/database-fjall/blocks.d.ts +25 -0
- package/packages/jam/database-fjall/blocks.d.ts.map +1 -0
- package/packages/jam/database-fjall/blocks.js +86 -0
- package/packages/jam/database-fjall/blocks.test.d.ts +2 -0
- package/packages/jam/database-fjall/blocks.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/blocks.test.js +62 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts +61 -10
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.js +131 -26
- package/packages/jam/database-fjall/hybrid-states.test.js +31 -1
- package/packages/jam/database-fjall/index.d.ts +2 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -1
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +29 -17
- package/packages/jam/database-fjall/root.d.ts.map +1 -1
- package/packages/jam/database-fjall/root.js +33 -13
- package/packages/jam/database-fjall/states.d.ts +31 -0
- package/packages/jam/database-fjall/states.d.ts.map +1 -0
- package/packages/jam/database-fjall/states.js +99 -0
- package/packages/jam/database-fjall/states.test.d.ts +2 -0
- package/packages/jam/database-fjall/states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/states.test.js +150 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +12 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/hybrid-states.js +45 -4
- package/packages/jam/database-lmdb/states.d.ts +1 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/states.js +5 -0
- package/packages/jam/jamnp-s/tasks/ticket-distribution.js +1 -1
- package/packages/jam/node/export.d.ts.map +1 -1
- package/packages/jam/node/export.js +18 -9
- package/packages/jam/node/main-fuzz.d.ts +4 -1
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +127 -55
- package/packages/jam/node/main-importer.d.ts +11 -2
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +27 -12
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +42 -15
- package/packages/jam/node/workers.d.ts +4 -4
- package/packages/jam/node/workers.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-vrf.d.ts +22 -2
- package/packages/jam/safrole/bandersnatch-vrf.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-vrf.js +54 -20
- package/packages/jam/safrole/bandersnatch-vrf.test.js +3 -2
- package/packages/jam/safrole/bandersnatch-wasm.d.ts +10 -0
- package/packages/jam/safrole/bandersnatch-wasm.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-wasm.js +12 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts +5 -4
- package/packages/jam/ticket-pool/ticket-validator.d.ts.map +1 -1
- package/packages/jam/ticket-pool/ticket-validator.js +8 -3
- package/packages/jam/ticket-pool/ticket-validator.test.js +5 -4
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts +2 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts.map +1 -1
- package/packages/jam/ticket-pool/verified-ticket-pool.js +4 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.js +5 -5
- package/packages/workers/api/config.d.ts +2 -2
- package/packages/workers/api/config.d.ts.map +1 -1
- package/packages/workers/api/config.js +1 -1
- package/packages/workers/api/port.d.ts +5 -1
- package/packages/workers/api/port.d.ts.map +1 -1
- package/packages/workers/api/port.js +56 -13
- package/packages/workers/api/port.test.js +35 -0
- package/packages/workers/api-node/config.d.ts +58 -9
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +140 -18
- package/packages/workers/api-node/config.test.js +52 -3
- package/packages/workers/api-node/port.d.ts +3 -0
- package/packages/workers/api-node/port.d.ts.map +1 -1
- package/packages/workers/api-node/port.js +31 -2
- package/packages/workers/api-node/port.test.js +17 -0
- package/packages/workers/api-node/protocol.d.ts +3 -3
- package/packages/workers/api-node/protocol.d.ts.map +1 -1
- package/packages/workers/api-node/protocol.js +18 -12
- package/packages/workers/block-authorship/{generator.d.ts → block-generator.d.ts} +5 -5
- package/packages/workers/block-authorship/block-generator.d.ts.map +1 -0
- package/packages/workers/block-authorship/{generator.js → block-generator.js} +3 -3
- package/packages/workers/block-authorship/block-generator.test.d.ts +2 -0
- package/packages/workers/block-authorship/block-generator.test.d.ts.map +1 -0
- package/packages/workers/block-authorship/{generator.test.js → block-generator.test.js} +8 -8
- package/packages/workers/block-authorship/epoch-authoring-slots.d.ts +35 -0
- package/packages/workers/block-authorship/epoch-authoring-slots.d.ts.map +1 -0
- package/packages/workers/block-authorship/epoch-authoring-slots.js +86 -0
- package/packages/workers/block-authorship/epoch-tracker.d.ts +29 -0
- package/packages/workers/block-authorship/epoch-tracker.d.ts.map +1 -0
- package/packages/workers/block-authorship/epoch-tracker.js +80 -0
- package/packages/workers/block-authorship/index.d.ts.map +1 -1
- package/packages/workers/block-authorship/index.js +1 -1
- package/packages/workers/block-authorship/main.d.ts +3 -0
- package/packages/workers/block-authorship/main.d.ts.map +1 -1
- package/packages/workers/block-authorship/main.js +194 -262
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts +2 -0
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.js +23 -0
- package/packages/workers/block-authorship/ticket-generator/index.d.ts +16 -0
- package/packages/workers/block-authorship/ticket-generator/index.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/index.js +62 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.d.ts +50 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.js +54 -0
- package/packages/workers/block-authorship/{ticket-generator.d.ts → ticket-generator/ticket-generator.d.ts} +4 -0
- package/packages/workers/block-authorship/ticket-generator/ticket-generator.d.ts.map +1 -0
- package/packages/workers/block-authorship/{ticket-generator.js → ticket-generator/ticket-generator.js} +19 -9
- package/packages/workers/block-authorship/ticket-generator/ticket-generator.test.d.ts.map +1 -0
- package/packages/workers/block-authorship/{ticket-generator.test.js → ticket-generator/ticket-generator.test.js} +13 -9
- package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts +36 -0
- package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/worker-pool.js +111 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts +7 -8
- package/packages/workers/block-authorship/ticket-validator.d.ts.map +1 -1
- package/packages/workers/block-authorship/ticket-validator.js +26 -23
- package/packages/workers/comms-authorship-network/protocol.d.ts +4 -4
- package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/protocol.js +4 -5
- package/packages/workers/comms-authorship-network/tickets-message.d.ts +0 -14
- package/packages/workers/comms-authorship-network/tickets-message.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/tickets-message.js +0 -17
- package/packages/workers/importer/finality.d.ts +2 -0
- package/packages/workers/importer/finality.d.ts.map +1 -1
- package/packages/workers/importer/finality.js +8 -1
- package/packages/workers/importer/importer.d.ts +2 -2
- package/packages/workers/importer/importer.d.ts.map +1 -1
- package/packages/workers/importer/importer.js +6 -5
- package/packages/workers/importer/main.d.ts +1 -1
- package/packages/workers/importer/main.d.ts.map +1 -1
- package/packages/workers/importer/main.js +1 -1
- package/packages/workers/importer/stats.d.ts +1 -3
- package/packages/workers/importer/stats.d.ts.map +1 -1
- package/packages/workers/importer/stats.js +12 -12
- package/packages/workers/jam-network/main.d.ts.map +1 -1
- package/packages/workers/jam-network/main.js +9 -4
- package/packages/workers/block-authorship/generator.d.ts.map +0 -1
- package/packages/workers/block-authorship/generator.test.d.ts +0 -2
- package/packages/workers/block-authorship/generator.test.d.ts.map +0 -1
- package/packages/workers/block-authorship/ticket-generator.d.ts.map +0 -1
- package/packages/workers/block-authorship/ticket-generator.test.d.ts.map +0 -1
- /package/packages/configs/{typeberry-dev.json → typeberry-dev-tiny.json} +0 -0
- /package/packages/workers/block-authorship/{ticket-generator.test.d.ts → ticket-generator/ticket-generator.test.d.ts} +0 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { afterEach, before, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
|
|
5
|
+
import { Bytes, BytesBlob } from "#@typeberry/bytes";
|
|
6
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
7
|
+
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
8
|
+
import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
|
|
9
|
+
import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
|
|
10
|
+
import { testState } from "#@typeberry/state/test.utils.js";
|
|
11
|
+
import { StateEntries } from "#@typeberry/state-merkleization";
|
|
12
|
+
import { deepEqual, OK, Result } from "#@typeberry/utils";
|
|
13
|
+
import { FjallRoot } from "./root.js";
|
|
14
|
+
import { FjallStates } from "./states.js";
|
|
15
|
+
let blake2b;
|
|
16
|
+
before(async () => {
|
|
17
|
+
blake2b = await Blake2b.createHasher();
|
|
18
|
+
});
|
|
19
|
+
describe("Fjall states database", () => {
|
|
20
|
+
const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
21
|
+
const spec = tinyChainSpec;
|
|
22
|
+
let tmpDir = "";
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
tmpDir = fs.mkdtempSync("typeberry-fjall-states-");
|
|
25
|
+
});
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
28
|
+
});
|
|
29
|
+
it("imports and reads an empty state", async () => {
|
|
30
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
31
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
32
|
+
try {
|
|
33
|
+
const emptyState = InMemoryState.empty(spec);
|
|
34
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
|
|
35
|
+
const emptyRoot = serialized.getRootHash(blake2b);
|
|
36
|
+
deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
|
|
37
|
+
const newState = states.getState(headerHash);
|
|
38
|
+
assert.ok(newState !== null);
|
|
39
|
+
assert.strictEqual(`${await states.getStateRoot(newState)}`, `${emptyRoot}`);
|
|
40
|
+
deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await states.close();
|
|
44
|
+
await root.close();
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
it("reads large values back from disk", async () => {
|
|
48
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
49
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
50
|
+
try {
|
|
51
|
+
const big1 = BytesBlob.blobFromString("x".repeat(100));
|
|
52
|
+
const big2 = BytesBlob.blobFromString("y".repeat(100));
|
|
53
|
+
const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
|
|
54
|
+
const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
55
|
+
const entries = StateEntries.fromEntriesUnsafe([
|
|
56
|
+
[key1, big1],
|
|
57
|
+
[key2, big2],
|
|
58
|
+
]);
|
|
59
|
+
deepEqual(await states.insertInitialState(headerHash, entries), Result.ok(OK));
|
|
60
|
+
const state = states.getState(headerHash);
|
|
61
|
+
assert.ok(state !== null);
|
|
62
|
+
assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
|
|
63
|
+
assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
await states.close();
|
|
67
|
+
await root.close();
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
it("updates state", async () => {
|
|
71
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
72
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
73
|
+
try {
|
|
74
|
+
const state = InMemoryState.empty(spec);
|
|
75
|
+
await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
|
|
76
|
+
const newState = states.getState(headerHash);
|
|
77
|
+
assert.ok(newState !== null);
|
|
78
|
+
const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
79
|
+
const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
|
|
80
|
+
const stateUpdate = {
|
|
81
|
+
timeslot: tryAsTimeSlot(15),
|
|
82
|
+
privilegedServices: PrivilegedServices.create({
|
|
83
|
+
manager: tryAsServiceId(1),
|
|
84
|
+
assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
|
|
85
|
+
delegator: tryAsServiceId(3),
|
|
86
|
+
registrar: tryAsServiceId(4),
|
|
87
|
+
autoAccumulateServices: new Map(),
|
|
88
|
+
}),
|
|
89
|
+
updated: new Map([
|
|
90
|
+
[
|
|
91
|
+
tryAsServiceId(1),
|
|
92
|
+
UpdateService.create({
|
|
93
|
+
serviceInfo: ServiceAccountInfo.create({
|
|
94
|
+
codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
|
|
95
|
+
balance: tryAsU64(1_000_000),
|
|
96
|
+
accumulateMinGas: tryAsServiceGas(10_000),
|
|
97
|
+
onTransferMinGas: tryAsServiceGas(5_000),
|
|
98
|
+
storageUtilisationBytes: tryAsU64(1_000),
|
|
99
|
+
gratisStorage: tryAsU64(0),
|
|
100
|
+
storageUtilisationCount: tryAsU32(1),
|
|
101
|
+
created: tryAsTimeSlot(0),
|
|
102
|
+
lastAccumulation: tryAsTimeSlot(0),
|
|
103
|
+
parentService: tryAsServiceId(0),
|
|
104
|
+
}),
|
|
105
|
+
lookupHistory,
|
|
106
|
+
}),
|
|
107
|
+
],
|
|
108
|
+
]),
|
|
109
|
+
};
|
|
110
|
+
deepEqual(state.applyUpdate(stateUpdate), Result.ok(OK));
|
|
111
|
+
deepEqual(await states.updateAndSetState(headerHash2, newState, stateUpdate), Result.ok(OK));
|
|
112
|
+
const updatedState = states.getState(headerHash2);
|
|
113
|
+
assert.ok(updatedState !== null);
|
|
114
|
+
deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
|
|
115
|
+
[
|
|
116
|
+
tryAsServiceId(1),
|
|
117
|
+
{
|
|
118
|
+
storageKeys: [],
|
|
119
|
+
preimages: [],
|
|
120
|
+
lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
])), state);
|
|
124
|
+
assert.strictEqual(`${await states.getStateRoot(updatedState)}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
await states.close();
|
|
128
|
+
await root.close();
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
it("imports a more complex state", async () => {
|
|
132
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
133
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
134
|
+
try {
|
|
135
|
+
const initialState = testState();
|
|
136
|
+
const initialService = initialState.services.get(tryAsServiceId(0));
|
|
137
|
+
assert.ok(initialService !== undefined);
|
|
138
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
|
|
139
|
+
deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
|
|
140
|
+
const newState = states.getState(headerHash);
|
|
141
|
+
assert.ok(newState !== null);
|
|
142
|
+
assert.strictEqual(`${await states.getStateRoot(newState)}`, `${serialized.getRootHash(blake2b)}`);
|
|
143
|
+
deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
await states.close();
|
|
147
|
+
await root.close();
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
|
|
|
12
12
|
* Reads go straight to lmdb, which keeps its own page cache.
|
|
13
13
|
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
14
14
|
* keep the heap usage bounded.
|
|
15
|
+
*
|
|
16
|
+
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
17
|
+
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
18
|
+
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
19
|
+
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
20
|
+
* run are never collected.
|
|
15
21
|
*/
|
|
16
22
|
export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
|
|
17
23
|
private readonly spec;
|
|
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
20
26
|
private readonly inMemStates;
|
|
21
27
|
private readonly lmdbValues;
|
|
22
28
|
private readonly valuesDb;
|
|
29
|
+
private readonly refsStore;
|
|
30
|
+
private readonly refs;
|
|
31
|
+
private pendingCleanup;
|
|
23
32
|
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
|
|
24
33
|
spec: ChainSpec;
|
|
25
34
|
blake2b: Blake2b;
|
|
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
33
42
|
updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
34
43
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
35
44
|
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
45
|
+
commitFinalized(headers: HeaderHash[]): void;
|
|
36
46
|
markUnused(header: HeaderHash): void;
|
|
47
|
+
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
48
|
+
private applyRefs;
|
|
37
49
|
diskSizeInBytes(): number | null;
|
|
38
50
|
close(): Promise<void>;
|
|
39
51
|
/** Write new large values to LMDB in one transaction. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,
|
|
1
|
+
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { HashDictionary, SortedSet } from "#@typeberry/collections";
|
|
2
|
-
import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
|
|
2
|
+
import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
|
|
3
|
+
import { Logger } from "#@typeberry/logger";
|
|
3
4
|
import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
|
|
4
5
|
import { leafComparator } from "#@typeberry/trie";
|
|
5
6
|
import { OK, Result } from "#@typeberry/utils";
|
|
6
7
|
import { LmdbRoot } from "./root.js";
|
|
8
|
+
const logger = Logger.new(import.meta.filename, "db");
|
|
7
9
|
/**
|
|
8
10
|
* Hybrid serialized-states db.
|
|
9
11
|
*
|
|
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
|
|
|
11
13
|
* Reads go straight to lmdb, which keeps its own page cache.
|
|
12
14
|
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
13
15
|
* keep the heap usage bounded.
|
|
16
|
+
*
|
|
17
|
+
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
18
|
+
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
19
|
+
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
20
|
+
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
21
|
+
* run are never collected.
|
|
14
22
|
*/
|
|
15
23
|
export class HybridSerializedStates {
|
|
16
24
|
spec;
|
|
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
|
|
|
20
28
|
lmdbValues;
|
|
21
29
|
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
22
30
|
valuesDb;
|
|
31
|
+
refsStore = new InMemoryValueRefsStore();
|
|
32
|
+
refs = new ValueRefs(this.refsStore);
|
|
33
|
+
// Queue of not-yet-committed value removals, awaited on close.
|
|
34
|
+
pendingCleanup = Promise.resolve();
|
|
23
35
|
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
|
|
24
36
|
const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
|
|
25
37
|
return new HybridSerializedStates(spec, blake2b, root);
|
|
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
|
|
|
38
50
|
return res;
|
|
39
51
|
}
|
|
40
52
|
this.inMemStates.set(headerHash, leafs);
|
|
53
|
+
this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
|
|
41
54
|
return Result.ok(OK);
|
|
42
55
|
}
|
|
43
56
|
async updateAndSetState(header, state, update) {
|
|
44
57
|
const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
|
|
45
58
|
// Clone the leaf set before mutating: the previous state keeps using its own.
|
|
46
59
|
const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
|
|
47
|
-
const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
60
|
+
const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
48
61
|
const res = await this.writeValues(values);
|
|
49
62
|
if (res.isError) {
|
|
50
63
|
// Leave the caller's state untouched: its new leaves would reference
|
|
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
|
|
|
55
68
|
// values are durably written.
|
|
56
69
|
state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
|
|
57
70
|
this.inMemStates.set(header, leafs);
|
|
71
|
+
this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
|
|
58
72
|
return Result.ok(OK);
|
|
59
73
|
}
|
|
60
74
|
async getStateRoot(state) {
|
|
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
|
|
|
68
82
|
const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
|
|
69
83
|
return SerializedState.new(this.spec, this.blake2b, leafDb);
|
|
70
84
|
}
|
|
85
|
+
commitFinalized(headers) {
|
|
86
|
+
this.applyRefs(this.refs.commitFinalized(headers));
|
|
87
|
+
}
|
|
71
88
|
markUnused(header) {
|
|
72
|
-
//
|
|
73
|
-
//
|
|
89
|
+
// Release the speculative references first (a no-op for finalized states,
|
|
90
|
+
// whose deltas were already consumed by `commitFinalized`).
|
|
91
|
+
this.applyRefs(this.refs.releaseUnfinalized(header));
|
|
74
92
|
this.inMemStates.delete(header);
|
|
75
93
|
}
|
|
94
|
+
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
95
|
+
applyRefs(update) {
|
|
96
|
+
this.refsStore.apply(update);
|
|
97
|
+
if (update.removeValues.length === 0) {
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
// Queued, not awaited: a failed removal only leaks a value.
|
|
101
|
+
this.pendingCleanup = this.pendingCleanup
|
|
102
|
+
.then(() => this.lmdbValues.transaction(() => {
|
|
103
|
+
for (const v of update.removeValues) {
|
|
104
|
+
this.lmdbValues.remove(v.raw);
|
|
105
|
+
}
|
|
106
|
+
}))
|
|
107
|
+
.catch((e) => {
|
|
108
|
+
logger.error `Failed to remove unreferenced values: ${e}`;
|
|
109
|
+
});
|
|
110
|
+
}
|
|
76
111
|
diskSizeInBytes() {
|
|
77
112
|
return this.root.sizeInBytes();
|
|
78
113
|
}
|
|
79
114
|
async close() {
|
|
115
|
+
await this.pendingCleanup;
|
|
80
116
|
await this.lmdbValues.close();
|
|
81
117
|
await this.root.close();
|
|
82
118
|
}
|
|
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
|
|
|
86
122
|
return Result.ok(OK);
|
|
87
123
|
}
|
|
88
124
|
try {
|
|
125
|
+
// Flush queued removals first: a pending cleanup might be about to delete
|
|
126
|
+
// a content hash we are re-inserting now. Writing behind it keeps the
|
|
127
|
+
// re-inserted value on disk (removals are only queued at refcount 0, so
|
|
128
|
+
// none can be queued for a value still referenced).
|
|
129
|
+
await this.pendingCleanup;
|
|
89
130
|
await this.lmdbValues.transaction(() => {
|
|
90
131
|
for (const [hash, val] of values) {
|
|
91
132
|
this.lmdbValues.put(hash.raw, val.raw);
|
|
@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
|
|
|
68
68
|
updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
69
69
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
70
70
|
getState(root: HeaderHash): SerializedState<LeafDb> | null;
|
|
71
|
+
commitFinalized(_headers: HeaderHash[]): void;
|
|
71
72
|
markUnused(header: HeaderHash): void;
|
|
72
73
|
diskSizeInBytes(): number | null;
|
|
73
74
|
close(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
|
|
@@ -128,6 +128,11 @@ export class LmdbStates {
|
|
|
128
128
|
}
|
|
129
129
|
return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
|
|
130
130
|
}
|
|
131
|
+
commitFinalized(_headers) {
|
|
132
|
+
// Values are never pruned here. This db survives restarts, so refcounting
|
|
133
|
+
// would need counts persisted (and crash-consistent) alongside the values.
|
|
134
|
+
// See the TODO above - not implemented until actually needed.
|
|
135
|
+
}
|
|
131
136
|
markUnused(header) {
|
|
132
137
|
this.states.removeSync(header.raw);
|
|
133
138
|
}
|
|
@@ -108,7 +108,7 @@ export class TicketDistributionTask {
|
|
|
108
108
|
// Wrap with Promise.resolve().then() so a synchronous throw inside the validator
|
|
109
109
|
// funnels into the same .catch() as an async rejection.
|
|
110
110
|
Promise.resolve()
|
|
111
|
-
.then(() => validator.validate(epochIndex, ticket))
|
|
111
|
+
.then(() => validator.validate(epochIndex, [ticket]))
|
|
112
112
|
.then((result) => {
|
|
113
113
|
if (result.isOk) {
|
|
114
114
|
this.addTicket(epochIndex, ticket);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}
|
|
@@ -3,9 +3,10 @@ import path from "node:path";
|
|
|
3
3
|
import { Block as BlockCodec } from "#@typeberry/block";
|
|
4
4
|
import { Bytes } from "#@typeberry/bytes";
|
|
5
5
|
import { Encoder } from "#@typeberry/codec";
|
|
6
|
+
import { RegularStateBackend } from "#@typeberry/config-node";
|
|
6
7
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
7
8
|
import { Logger } from "#@typeberry/logger";
|
|
8
|
-
import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
|
|
9
|
+
import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
|
|
9
10
|
import { getChainSpec, getDatabasePath } from "./common.js";
|
|
10
11
|
export async function exportBlocks(jamNodeConfig, output, withRelPath) {
|
|
11
12
|
const logger = Logger.new(import.meta.filename, "export");
|
|
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
|
|
|
29
30
|
const blake2b = await Blake2b.createHasher();
|
|
30
31
|
const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
|
|
31
32
|
const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
|
|
32
|
-
const config =
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
|
|
34
|
+
? FjallWorkerConfig.new({
|
|
35
|
+
nodeName: jamNodeConfig.nodeName,
|
|
36
|
+
chainSpec,
|
|
37
|
+
blake2b,
|
|
38
|
+
dbPath,
|
|
39
|
+
workerParams: undefined,
|
|
40
|
+
})
|
|
41
|
+
: LmdbWorkerConfig.new({
|
|
42
|
+
nodeName: jamNodeConfig.nodeName,
|
|
43
|
+
chainSpec,
|
|
44
|
+
blake2b,
|
|
45
|
+
dbPath,
|
|
46
|
+
workerParams: undefined,
|
|
47
|
+
});
|
|
48
|
+
const rootDb = await config.openDatabase();
|
|
40
49
|
const blocks = rootDb.getBlocksDb();
|
|
41
50
|
logger.info `📖 Gathering blocks...`;
|
|
42
51
|
const hashes = [];
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type FuzzVersion } from "#@typeberry/ext-ipc";
|
|
2
2
|
import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
|
|
3
|
+
import { type Closer } from "#@typeberry/utils";
|
|
3
4
|
import type { JamConfig } from "./jam-config.js";
|
|
4
5
|
export type FuzzConfig = {
|
|
5
6
|
version: FuzzVersion;
|
|
@@ -24,5 +25,7 @@ export declare function getFuzzDetails(): {
|
|
|
24
25
|
nodeVersion: fuzzV1.Version;
|
|
25
26
|
gpVersion: fuzzV1.Version;
|
|
26
27
|
};
|
|
27
|
-
export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<
|
|
28
|
+
export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<{
|
|
29
|
+
close: Closer;
|
|
30
|
+
}>;
|
|
28
31
|
//# sourceMappingURL=main-fuzz.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}
|
|
@@ -8,7 +8,7 @@ import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
|
|
|
8
8
|
import { HASH_SIZE } from "#@typeberry/hash";
|
|
9
9
|
import { Logger } from "#@typeberry/logger";
|
|
10
10
|
import { CURRENT_VERSION, Result, version } from "#@typeberry/utils";
|
|
11
|
-
import { logHostEnvironment } from "#@typeberry/workers-api-node";
|
|
11
|
+
import { FjallValuesSession, logHostEnvironment } from "#@typeberry/workers-api-node";
|
|
12
12
|
import { getChainSpec } from "./common.js";
|
|
13
13
|
import { mainImporter } from "./main-importer.js";
|
|
14
14
|
const logger = Logger.new(import.meta.filename, "fuzztarget");
|
|
@@ -17,6 +17,14 @@ const FUZZ_DB_SUBDIR = "typeberry-fuzz-db";
|
|
|
17
17
|
const FUZZ_DB_FJALL = "fjall-hybrid";
|
|
18
18
|
const FUZZ_DB_LMDB = "lmdb-hybrid";
|
|
19
19
|
const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, FUZZ_DB_LMDB];
|
|
20
|
+
/** Subdirectory (under the fuzzer's db dir) holding the reused fjall values keyspace. */
|
|
21
|
+
const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
|
|
22
|
+
/**
|
|
23
|
+
* Size of the fjall block-cache for the fuzz session. Values pile up across
|
|
24
|
+
* resets (for fjall we do not wipe between them), so this cache is what keeps
|
|
25
|
+
* the resident memory bounded.
|
|
26
|
+
*/
|
|
27
|
+
const FUZZ_FJALL_CACHE_BYTES = 128 * 1024 * 1024;
|
|
20
28
|
/**
|
|
21
29
|
* Resolve the directory the fuzzer should use for its on-disk database, or
|
|
22
30
|
* `undefined` for an in-memory database. The dedicated `FUZZ_DB_SUBDIR` is
|
|
@@ -63,6 +71,14 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
63
71
|
logger.info `🗄️ Fuzz persistent backend: ${hybridStateBackend}.`;
|
|
64
72
|
}
|
|
65
73
|
let runningNode = null;
|
|
74
|
+
// The fjall values keyspace is opened once per fuzz session and reused on
|
|
75
|
+
// every reset, because opening it is the slow part. Only the in-memory blocks
|
|
76
|
+
// and leaf sets are rebuilt for each vector. fjall-hybrid only.
|
|
77
|
+
let fjallSession = null;
|
|
78
|
+
// Set when close() starts. Guards resetState so a fuzz command arriving
|
|
79
|
+
// mid-shutdown can't build a fresh node that close() then orphans.
|
|
80
|
+
let isClosing = false;
|
|
81
|
+
let activeReset = null;
|
|
66
82
|
const chainSpec = getChainSpec(config.node.flavor);
|
|
67
83
|
const closeFuzzTarget = startFuzzTarget(fuzzConfig.version, fuzzConfig.socket, {
|
|
68
84
|
...getFuzzDetails(),
|
|
@@ -86,66 +102,122 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
86
102
|
}
|
|
87
103
|
return runningNode.getStateEntries(hash);
|
|
88
104
|
},
|
|
89
|
-
resetState:
|
|
105
|
+
resetState: (header, state, ancestry) => {
|
|
106
|
+
const reset = (async () => {
|
|
107
|
+
if (isClosing) {
|
|
108
|
+
return Bytes.zero(HASH_SIZE).asOpaque();
|
|
109
|
+
}
|
|
110
|
+
if (runningNode !== null) {
|
|
111
|
+
const finish = runningNode.close();
|
|
112
|
+
runningNode = null;
|
|
113
|
+
await finish;
|
|
114
|
+
}
|
|
115
|
+
const buildNode = (databaseBasePath) => {
|
|
116
|
+
const isPersistent = databaseBasePath !== undefined;
|
|
117
|
+
return mainImporter({
|
|
118
|
+
...config,
|
|
119
|
+
node: {
|
|
120
|
+
...config.node,
|
|
121
|
+
databaseBasePath,
|
|
122
|
+
chainSpec: {
|
|
123
|
+
...config.node.chainSpec,
|
|
124
|
+
genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
|
|
125
|
+
genesisState: new Map(state),
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
ancestry,
|
|
129
|
+
network: null,
|
|
130
|
+
}, withRelPath, {
|
|
131
|
+
initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
|
|
132
|
+
// Hybrid keeps leaf sets in RAM, so they must be windowed exactly
|
|
133
|
+
// like the in-memory backend; only the large values live on disk.
|
|
134
|
+
dummyFinalityDepth: 20,
|
|
135
|
+
pruneBlocks: true,
|
|
136
|
+
// The on-disk fuzz db is throwaway (we wipe it), so open it ephemeral and
|
|
137
|
+
// skip the fsync, we do not need durability here. On full spec ephemeral
|
|
138
|
+
// also turns on compression further down, so the big values do not grow the
|
|
139
|
+
// db too much. Tiny stays uncompressed, its db is small and speed matters more.
|
|
140
|
+
ephemeral: isPersistent,
|
|
141
|
+
stateBackend: isPersistent ? hybridStateBackend : "lmdb",
|
|
142
|
+
// Reuse the session keyspace (fjall-hybrid only, other backends
|
|
143
|
+
// ignore it). Nothing to pass for the in-memory fallback.
|
|
144
|
+
sharedFjallSession: isPersistent ? (fjallSession ?? undefined) : undefined,
|
|
145
|
+
});
|
|
146
|
+
};
|
|
147
|
+
if (fuzzDbBase !== undefined) {
|
|
148
|
+
try {
|
|
149
|
+
if (hybridStateBackend === FUZZ_DB_FJALL) {
|
|
150
|
+
// fjall-hybrid: open the values keyspace once and reuse it on every
|
|
151
|
+
// reset. The values partition is content-addressed and immutable, so
|
|
152
|
+
// it is fine that values pile up across resets, the unreferenced ones
|
|
153
|
+
// just sit there. `initializeDatabase` decides whether the db is
|
|
154
|
+
// already initialized from the in-memory blocks, which we rebuild on
|
|
155
|
+
// every reset, not from the values store, so reusing it does not
|
|
156
|
+
// resume the previous run.
|
|
157
|
+
if (fjallSession === null) {
|
|
158
|
+
// Start from a clean slate once, then keep the keyspace open.
|
|
159
|
+
await wipeFuzzDb(fuzzDbBase);
|
|
160
|
+
fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
|
|
161
|
+
ephemeral: true,
|
|
162
|
+
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
163
|
+
});
|
|
164
|
+
logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
// lmdb-hybrid: keep the old behaviour, wipe and reopen on every
|
|
169
|
+
// reset. A fresh db each reset makes `initializeDatabase` set up
|
|
170
|
+
// genesis again instead of resuming the previous run.
|
|
171
|
+
await wipeFuzzDb(fuzzDbBase);
|
|
172
|
+
}
|
|
173
|
+
runningNode = await buildNode(fuzzDbBase);
|
|
174
|
+
return await runningNode.getBestStateRootHash();
|
|
175
|
+
}
|
|
176
|
+
catch (e) {
|
|
177
|
+
// A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
|
|
178
|
+
logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
|
|
179
|
+
runningNode = null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
runningNode = await buildNode(undefined);
|
|
183
|
+
return await runningNode.getBestStateRootHash();
|
|
184
|
+
})();
|
|
185
|
+
activeReset = reset;
|
|
186
|
+
const clearActiveReset = () => {
|
|
187
|
+
if (activeReset === reset) {
|
|
188
|
+
activeReset = null;
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
reset.then(clearActiveReset, clearActiveReset);
|
|
192
|
+
return reset;
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
return {
|
|
196
|
+
close: async () => {
|
|
197
|
+
isClosing = true;
|
|
198
|
+
// Stop accepting connections + unlink the socket.
|
|
199
|
+
closeFuzzTarget();
|
|
200
|
+
// Drain the active session (flush + close DB). Swallow errors so a
|
|
201
|
+
// failing close still lets the process exit 0; the db is wiped next.
|
|
202
|
+
// The node references the shared fjall session, so it must close first.
|
|
203
|
+
if (activeReset !== null) {
|
|
204
|
+
await activeReset.catch((e) => logger.error `Error waiting for fuzz reset: ${e}`);
|
|
205
|
+
}
|
|
90
206
|
if (runningNode !== null) {
|
|
91
|
-
const
|
|
207
|
+
const node = runningNode;
|
|
92
208
|
runningNode = null;
|
|
93
|
-
await
|
|
209
|
+
await node.close().catch((e) => logger.error `Error closing fuzz node: ${e}`);
|
|
210
|
+
}
|
|
211
|
+
// Release the reused fjall values keyspace before wiping its files.
|
|
212
|
+
if (fjallSession !== null) {
|
|
213
|
+
const session = fjallSession;
|
|
214
|
+
fjallSession = null;
|
|
215
|
+
await session.close().catch((e) => logger.error `Error closing fjall session: ${e}`);
|
|
94
216
|
}
|
|
95
|
-
const buildNode = (databaseBasePath) => {
|
|
96
|
-
const isPersistent = databaseBasePath !== undefined;
|
|
97
|
-
return mainImporter({
|
|
98
|
-
...config,
|
|
99
|
-
node: {
|
|
100
|
-
...config.node,
|
|
101
|
-
databaseBasePath,
|
|
102
|
-
chainSpec: {
|
|
103
|
-
...config.node.chainSpec,
|
|
104
|
-
genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
|
|
105
|
-
genesisState: new Map(state),
|
|
106
|
-
},
|
|
107
|
-
},
|
|
108
|
-
ancestry,
|
|
109
|
-
network: null,
|
|
110
|
-
}, withRelPath, {
|
|
111
|
-
initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
|
|
112
|
-
// Hybrid keeps leaf sets in RAM, so they must be windowed exactly
|
|
113
|
-
// like the in-memory backend; only the large values live on disk.
|
|
114
|
-
dummyFinalityDepth: 20,
|
|
115
|
-
pruneBlocks: true,
|
|
116
|
-
// Long full-spec sessions accumulate a large, never-pruned values db.
|
|
117
|
-
// Syncing lets the OS reclaim dirty mmap pages, and compression (full
|
|
118
|
-
// spec only, where values are big) bounds its on-disk/page-cache size.
|
|
119
|
-
// Tiny stays uncompressed since its db is small and speed matters more.
|
|
120
|
-
ephemeral: isPersistent,
|
|
121
|
-
stateBackend: isPersistent ? hybridStateBackend : "lmdb",
|
|
122
|
-
});
|
|
123
|
-
};
|
|
124
217
|
if (fuzzDbBase !== undefined) {
|
|
125
|
-
|
|
126
|
-
// so the on-disk db must be empty: otherwise initializeDatabase sees an
|
|
127
|
-
// already-initialized db and silently resumes the previous run's state.
|
|
128
|
-
await wipeFuzzDb(fuzzDbBase);
|
|
129
|
-
try {
|
|
130
|
-
runningNode = await buildNode(fuzzDbBase);
|
|
131
|
-
return await runningNode.getBestStateRootHash();
|
|
132
|
-
}
|
|
133
|
-
catch (e) {
|
|
134
|
-
// A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
|
|
135
|
-
logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
|
|
136
|
-
runningNode = null;
|
|
137
|
-
}
|
|
218
|
+
await wipeFuzzDb(fuzzDbBase).catch(() => { });
|
|
138
219
|
}
|
|
139
|
-
runningNode = await buildNode(undefined);
|
|
140
|
-
return await runningNode.getBestStateRootHash();
|
|
141
220
|
},
|
|
142
|
-
});
|
|
143
|
-
return () => {
|
|
144
|
-
closeFuzzTarget();
|
|
145
|
-
if (fuzzDbBase !== undefined) {
|
|
146
|
-
// best-effort cleanup on shutdown; ignore failures (dir may already be gone).
|
|
147
|
-
wipeFuzzDb(fuzzDbBase).catch(() => { });
|
|
148
|
-
}
|
|
149
221
|
};
|
|
150
222
|
}
|
|
151
223
|
function isValidStateBackend(val) {
|