@typeberry/lib 0.8.4-9587022 → 0.8.4-faebc7a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -4
- package/packages/jam/database/states.d.ts +7 -0
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.d.ts +45 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.js +113 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts +2 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.test.js +83 -0
- package/packages/jam/database-fjall/index.d.ts +3 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -0
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +52 -0
- package/packages/jam/database-fjall/root.d.ts.map +1 -0
- package/packages/jam/database-fjall/root.js +85 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +3 -1
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/hybrid-states.js +5 -2
- package/packages/jam/database-lmdb/root.d.ts +14 -1
- package/packages/jam/database-lmdb/root.d.ts.map +1 -1
- package/packages/jam/database-lmdb/root.js +25 -5
- package/packages/jam/database-lmdb/states.d.ts +1 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/states.js +3 -0
- package/packages/jam/database-lmdb/states.test.js +4 -4
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts +18 -10
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map +1 -1
- package/packages/jam/jamnp-s/tasks/ticket-distribution.js +44 -68
- package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js +30 -8
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +21 -4
- package/packages/jam/node/main-importer.d.ts +7 -4
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +10 -4
- package/packages/jam/safrole/bandersnatch-vrf.d.ts +2 -2
- package/packages/jam/safrole/bandersnatch-vrf.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-vrf.js +9 -6
- package/packages/jam/safrole/bandersnatch-vrf.test.js +9 -7
- package/packages/jam/safrole/safrole.js +5 -5
- package/packages/jam/safrole/safrole.test.js +13 -13
- package/packages/jam/ticket-pool/index.d.ts +4 -0
- package/packages/jam/ticket-pool/index.d.ts.map +1 -0
- package/packages/jam/ticket-pool/index.js +3 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts +30 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.js +56 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.js +67 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts +46 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.js +29 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts +2 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.test.js +34 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts +24 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.js +37 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.js +54 -0
- package/packages/workers/api-node/config.d.ts +14 -5
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +29 -20
- package/packages/workers/api-node/config.test.js +38 -1
- package/packages/workers/block-authorship/main.d.ts.map +1 -1
- package/packages/workers/block-authorship/main.js +22 -72
- package/packages/workers/block-authorship/ticket-validator.d.ts +32 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-validator.js +56 -0
- package/packages/workers/comms-authorship-network/protocol.d.ts +10 -0
- package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/protocol.js +8 -1
- package/packages/workers/importer/importer.d.ts +1 -1
- package/packages/workers/importer/importer.d.ts.map +1 -1
- package/packages/workers/importer/importer.js +2 -2
- package/packages/workers/importer/{events-logger.d.ts → stats.d.ts} +7 -2
- package/packages/workers/importer/stats.d.ts.map +1 -0
- package/packages/workers/importer/{events-logger.js → stats.js} +21 -2
- package/packages/workers/jam-network/main.d.ts.map +1 -1
- package/packages/workers/jam-network/main.js +20 -4
- package/packages/workers/importer/events-logger.d.ts.map +0 -1
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Decoder, Encoder } from "#@typeberry/codec";
|
|
2
2
|
import { ChainSpec } from "#@typeberry/config";
|
|
3
3
|
import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
|
|
4
|
-
import { HybridSerializedStates
|
|
4
|
+
import { HybridSerializedStates as FjallHybridSerializedStates } from "#@typeberry/database-fjall";
|
|
5
|
+
import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
|
|
5
6
|
import { Blake2b } from "#@typeberry/hash";
|
|
6
7
|
import { ThreadPort } from "./port.js";
|
|
7
8
|
/** A worker config that's usable in node.js and uses LMDB database backend. */
|
|
@@ -32,9 +33,9 @@ export class LmdbWorkerConfig {
|
|
|
32
33
|
});
|
|
33
34
|
}
|
|
34
35
|
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
35
|
-
// When set, the underlying LMDB skips fsync
|
|
36
|
-
//
|
|
37
|
-
//
|
|
36
|
+
// When set, the underlying LMDB skips fsync. Only safe for throwaway
|
|
37
|
+
// databases (the fuzz target wipes on reset). Not transferred to worker
|
|
38
|
+
// threads, so the durable main node path always gets the default.
|
|
38
39
|
ephemeral = false) {
|
|
39
40
|
this.nodeName = nodeName;
|
|
40
41
|
this.chainSpec = chainSpec;
|
|
@@ -45,7 +46,10 @@ export class LmdbWorkerConfig {
|
|
|
45
46
|
this.ephemeral = ephemeral;
|
|
46
47
|
}
|
|
47
48
|
openDatabase(options = { readonly: true }) {
|
|
48
|
-
const lmdb = LmdbRoot.new(this.dbPath,
|
|
49
|
+
const lmdb = LmdbRoot.new(this.dbPath, {
|
|
50
|
+
readOnly: options.readonly,
|
|
51
|
+
ephemeral: this.ephemeral,
|
|
52
|
+
});
|
|
49
53
|
return {
|
|
50
54
|
getBlocksDb: () => LmdbBlocks.new(this.chainSpec, lmdb),
|
|
51
55
|
getStatesDb: () => LmdbStates.new(this.chainSpec, this.blake2b, lmdb),
|
|
@@ -107,11 +111,15 @@ export class InMemWorkerConfig {
|
|
|
107
111
|
}
|
|
108
112
|
/**
|
|
109
113
|
* Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
|
|
110
|
-
* but large values persisted to LMDB.
|
|
114
|
+
* but large values persisted to disk (LMDB or fjall, selected by `backend`).
|
|
115
|
+
*
|
|
116
|
+
* The fjall backend is opt-in so its performance can be compared against LMDB
|
|
117
|
+
* before committing to it. fjall opens its keyspace asynchronously, hence the
|
|
118
|
+
* async `new`.
|
|
111
119
|
*
|
|
112
120
|
* Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
|
|
113
121
|
* open/close/reopen dance that genesis init performs, so `openDatabase`
|
|
114
|
-
* returns the same instances and a no-op close. The
|
|
122
|
+
* returns the same instances and a no-op close. The values store is opened once
|
|
115
123
|
* here and closed by `HybridSerializedStates.close()` at importer teardown.
|
|
116
124
|
*
|
|
117
125
|
* In-process only: it holds shared mutable state (the in-memory leaf
|
|
@@ -125,33 +133,34 @@ export class HybridWorkerConfig {
|
|
|
125
133
|
blake2b;
|
|
126
134
|
dbPath;
|
|
127
135
|
ephemeral;
|
|
128
|
-
|
|
129
|
-
|
|
136
|
+
compression;
|
|
137
|
+
states;
|
|
138
|
+
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", }) {
|
|
139
|
+
// fjall opens its keyspace asynchronously; LMDB is synchronous. Either way
|
|
140
|
+
// the values store is created once here and shared across reopen.
|
|
141
|
+
const states = backend === "fjall"
|
|
142
|
+
? await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
|
|
143
|
+
: LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
|
|
144
|
+
return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
|
|
130
145
|
}
|
|
131
146
|
blocks;
|
|
132
|
-
states
|
|
133
|
-
constructor(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral) {
|
|
147
|
+
constructor(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states) {
|
|
134
148
|
this.nodeName = nodeName;
|
|
135
149
|
this.chainSpec = chainSpec;
|
|
136
150
|
this.workerParams = workerParams;
|
|
137
151
|
this.blake2b = blake2b;
|
|
138
152
|
this.dbPath = dbPath;
|
|
139
153
|
this.ephemeral = ephemeral;
|
|
154
|
+
this.compression = compression;
|
|
155
|
+
this.states = states;
|
|
140
156
|
this.blocks = InMemoryBlocks.new();
|
|
141
|
-
this.states = HybridSerializedStates.new({
|
|
142
|
-
spec: this.chainSpec,
|
|
143
|
-
blake2b: this.blake2b,
|
|
144
|
-
dbPath: this.dbPath,
|
|
145
|
-
ephemeral: this.ephemeral,
|
|
146
|
-
readOnly: false,
|
|
147
|
-
});
|
|
148
157
|
}
|
|
149
158
|
openDatabase(_options = { readonly: true }) {
|
|
150
159
|
return {
|
|
151
160
|
getBlocksDb: () => this.blocks,
|
|
152
161
|
getStatesDb: () => this.states,
|
|
153
|
-
// Leaf sets and blocks live in memory; the
|
|
154
|
-
//
|
|
162
|
+
// Leaf sets and blocks live in memory; the values store is closed via
|
|
163
|
+
// states.close() at importer teardown, so this is a no-op.
|
|
155
164
|
close: async () => { },
|
|
156
165
|
};
|
|
157
166
|
}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
2
3
|
import { describe, it } from "node:test";
|
|
3
4
|
import { MessageChannel } from "node:worker_threads";
|
|
4
5
|
import { codec } from "#@typeberry/codec";
|
|
5
6
|
import { tinyChainSpec } from "#@typeberry/config";
|
|
6
7
|
import { Blake2b } from "#@typeberry/hash";
|
|
7
8
|
import { tryAsU32 } from "#@typeberry/numbers";
|
|
8
|
-
import { configTransferList, LmdbWorkerConfig } from "./config.js";
|
|
9
|
+
import { configTransferList, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
|
|
9
10
|
import { ThreadPort } from "./port.js";
|
|
10
11
|
const spec = tinyChainSpec;
|
|
11
12
|
describe("LmdbWorkerConfig transfer list", () => {
|
|
@@ -39,3 +40,39 @@ describe("LmdbWorkerConfig transfer list", () => {
|
|
|
39
40
|
}
|
|
40
41
|
});
|
|
41
42
|
});
|
|
43
|
+
describe("HybridWorkerConfig", () => {
|
|
44
|
+
// Both persistent backends must construct asynchronously and hand out a
|
|
45
|
+
// working db. fjall is the experimental backend we want to benchmark.
|
|
46
|
+
for (const backend of ["lmdb", "fjall"]) {
|
|
47
|
+
it(`constructs and opens a ${backend}-backed hybrid db`, async () => {
|
|
48
|
+
const blake2b = await Blake2b.createHasher();
|
|
49
|
+
const dbPath = fs.mkdtempSync(`typeberry-hybrid-${backend}-`);
|
|
50
|
+
try {
|
|
51
|
+
const config = await HybridWorkerConfig.new({
|
|
52
|
+
nodeName: "node",
|
|
53
|
+
chainSpec: spec,
|
|
54
|
+
workerParams: undefined,
|
|
55
|
+
blake2b,
|
|
56
|
+
dbPath,
|
|
57
|
+
ephemeral: true,
|
|
58
|
+
backend,
|
|
59
|
+
});
|
|
60
|
+
const db = config.openDatabase({ readonly: false });
|
|
61
|
+
const states = db.getStatesDb();
|
|
62
|
+
try {
|
|
63
|
+
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
64
|
+
assert.notStrictEqual(states, undefined);
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
// The values store owns the on-disk resources (the no-op db.close()
|
|
68
|
+
// does not), so close it explicitly to release the fjall keyspace.
|
|
69
|
+
await states.close();
|
|
70
|
+
await db.close();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/main.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/main.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qCAAqC,CAAC;AAkB3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,KAAK,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAM9E,KAAK,MAAM,GAAG,YAAY,CAAC,qBAAqB,CAAC,CAAC;AAwBlD,wBAAsB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,eAAe,iBAuWpG"}
|
|
@@ -13,9 +13,11 @@ import bandersnatchVrf from "#@typeberry/safrole/bandersnatch-vrf.js";
|
|
|
13
13
|
import { BandernsatchWasm } from "#@typeberry/safrole/bandersnatch-wasm.js";
|
|
14
14
|
import { JAM_FALLBACK_SEAL, JAM_TICKET_SEAL } from "#@typeberry/safrole/constants.js";
|
|
15
15
|
import { SafroleSealingKeysKind } from "#@typeberry/state";
|
|
16
|
+
import { VerifiedTicketPool } from "#@typeberry/ticket-pool";
|
|
16
17
|
import { asOpaqueType, Result } from "#@typeberry/utils";
|
|
17
18
|
import { Generator } from "./generator.js";
|
|
18
19
|
import { generateTickets } from "./ticket-generator.js";
|
|
20
|
+
import { BandersnatchTicketValidator } from "./ticket-validator.js";
|
|
19
21
|
const logger = Logger.new(import.meta.filename, "author");
|
|
20
22
|
export async function main(config, comms, networkingComms) {
|
|
21
23
|
await initWasm();
|
|
@@ -181,78 +183,16 @@ export async function main(config, comms, networkingComms) {
|
|
|
181
183
|
}
|
|
182
184
|
return Result.ok(state.sealingKeySeries);
|
|
183
185
|
}
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
const
|
|
187
|
-
const
|
|
188
|
-
/**
|
|
189
|
-
* Adds pre-verified tickets to the in-memory ticket pool for the given epoch.
|
|
190
|
-
*
|
|
191
|
-
* Clears the pool when the epoch changes (we only ever need tickets for one epoch at a time).
|
|
192
|
-
* Deduplicates by ticket ID using a HashSet for O(1) lookup — prevents double-counting
|
|
193
|
-
* tickets received from multiple peers or via both CE-131 and CE-132 paths.
|
|
194
|
-
*/
|
|
195
|
-
function addToPool(epochIndex, verifiedTickets) {
|
|
196
|
-
if (ticketPool.size > 0 && !ticketPool.has(epochIndex)) {
|
|
197
|
-
ticketPool.clear();
|
|
198
|
-
ticketIdSets.clear();
|
|
199
|
-
}
|
|
200
|
-
const existing = ticketPool.get(epochIndex) ?? [];
|
|
201
|
-
let idSet = ticketIdSets.get(epochIndex) ?? null;
|
|
202
|
-
if (idSet === null) {
|
|
203
|
-
idSet = HashSet.new();
|
|
204
|
-
ticketIdSets.set(epochIndex, idSet);
|
|
205
|
-
}
|
|
206
|
-
for (const entry of verifiedTickets) {
|
|
207
|
-
if (!idSet.has(entry.id)) {
|
|
208
|
-
existing.push(entry);
|
|
209
|
-
idSet.insert(entry.id);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
ticketPool.set(epochIndex, existing);
|
|
213
|
-
}
|
|
214
|
-
/**
|
|
215
|
-
* Returns the correct tickets entropy for verification given the current state.
|
|
216
|
-
*
|
|
217
|
-
* When `state` is from epoch E-1 (i.e. we haven't produced epoch E's first block yet),
|
|
218
|
-
* the ticket entropy for epoch E is at index 1 (not yet shifted).
|
|
219
|
-
* After the epoch transition it moves to index 2.
|
|
220
|
-
*/
|
|
221
|
-
function getTicketEntropy(epochIndex, state) {
|
|
222
|
-
const stateEpoch = Math.floor(state.timeslot / chainSpec.epochLength);
|
|
223
|
-
return epochIndex > stateEpoch ? state.entropy[1] : state.entropy[2];
|
|
224
|
-
}
|
|
225
|
-
/**
|
|
226
|
-
* Verifies tickets against the ring commitment and current epoch entropy, then adds valid
|
|
227
|
-
* ones to the pool with their computed IDs.
|
|
228
|
-
*
|
|
229
|
-
* Called both for own generated tickets and for tickets relayed from peers.
|
|
230
|
-
* Verification computes the ticket ID (entropyHash) which is then used for
|
|
231
|
-
* deduplication in the pool and later when building the extrinsic.
|
|
232
|
-
*/
|
|
233
|
-
async function verifyAndAddToPool(epochIndex, tickets, state) {
|
|
234
|
-
const results = await bandersnatchVrf.verifyTickets(bandersnatch, state.designatedValidatorData.length, state.epochRoot, tickets, getTicketEntropy(epochIndex, state));
|
|
235
|
-
if (results.length !== tickets.length) {
|
|
236
|
-
logger.error `verifyTickets returned ${results.length} results for ${tickets.length} tickets`;
|
|
237
|
-
return false;
|
|
238
|
-
}
|
|
239
|
-
const verified = tickets
|
|
240
|
-
.map((ticket, i) => ({ ticket, id: results[i].entropyHash }))
|
|
241
|
-
.filter((_, i) => results[i].isValid);
|
|
242
|
-
addToPool(epochIndex, verified);
|
|
243
|
-
return verified.length > 0;
|
|
244
|
-
}
|
|
186
|
+
// Verified tickets for the current epoch, keyed by entropy hash (ticket id).
|
|
187
|
+
// Tickets enter via `validator.validate(...)` which both verifies and inserts.
|
|
188
|
+
const verifiedPool = new VerifiedTicketPool();
|
|
189
|
+
const ticketValidator = new BandersnatchTicketValidator(bandersnatch, chainSpec, verifiedPool, () => states.getState(blocks.getBestHeaderHash()));
|
|
245
190
|
// Receive a single ticket from peers (via jam-network worker).
|
|
246
191
|
// Returns true if the ticket passed validation so jam-network can decide whether to redistribute it.
|
|
247
192
|
networkingComms.setOnReceivedTickets(async ({ epochIndex, ticket }) => {
|
|
248
193
|
logger.log `Received ticket from peer for epoch ${epochIndex}`;
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
if (state === null) {
|
|
252
|
-
logger.warn `Cannot verify received ticket: no state available`;
|
|
253
|
-
return false;
|
|
254
|
-
}
|
|
255
|
-
return await verifyAndAddToPool(epochIndex, [ticket], state);
|
|
194
|
+
const result = await ticketValidator.validate(epochIndex, ticket);
|
|
195
|
+
return result.isOk;
|
|
256
196
|
});
|
|
257
197
|
const isFastForward = config.workerParams.isFastForward;
|
|
258
198
|
let lastGeneratedSlot = startTimeSlot;
|
|
@@ -301,8 +241,10 @@ export async function main(config, comms, networkingComms) {
|
|
|
301
241
|
}
|
|
302
242
|
else {
|
|
303
243
|
logger.log `Generated ${ticketsResult.ok.length} tickets for epoch ${epoch}. Distributing...`;
|
|
304
|
-
// Verify own tickets
|
|
305
|
-
|
|
244
|
+
// Verify own tickets (validator stores them in the pool with computed ids).
|
|
245
|
+
for (const ticket of ticketsResult.ok) {
|
|
246
|
+
await ticketValidator.validate(epoch, ticket);
|
|
247
|
+
}
|
|
306
248
|
// Send directly to network worker (bypasses main thread)
|
|
307
249
|
await networkingComms.sendTickets({ epochIndex: epoch, tickets: ticketsResult.ok });
|
|
308
250
|
}
|
|
@@ -328,6 +270,12 @@ export async function main(config, comms, networkingComms) {
|
|
|
328
270
|
}
|
|
329
271
|
await buildTicketAuthorshipCache(selingKeySeriesResult.ok, entropy);
|
|
330
272
|
}
|
|
273
|
+
// On every epoch boundary, push the authoritative ticket pool to networking so it
|
|
274
|
+
// can replace its redistribution set; this keeps the two sides from drifting.
|
|
275
|
+
if (isNewEpoch) {
|
|
276
|
+
const dumpTickets = verifiedPool.getForEpoch(epoch).map((entry) => entry.ticket);
|
|
277
|
+
await networkingComms.sendReplaceTicketPool({ epochIndex: epoch, tickets: dumpTickets });
|
|
278
|
+
}
|
|
331
279
|
const sealData = getSealData(selingKeySeriesResult.ok, keys, timeSlot, entropy);
|
|
332
280
|
if (sealData !== null && currentValidatorData !== null) {
|
|
333
281
|
const { key, sealPayload } = sealData;
|
|
@@ -336,8 +284,10 @@ export async function main(config, comms, networkingComms) {
|
|
|
336
284
|
continue;
|
|
337
285
|
}
|
|
338
286
|
logger.log `Attempting to create a block using ${sealData.logId} located at validator index ${validatorIndex}.`;
|
|
339
|
-
const currentEpochTickets =
|
|
340
|
-
const newBlock = await generator.nextBlockView(validatorIndex, key.bandersnatchSecret, sealPayload, timeSlot,
|
|
287
|
+
const currentEpochTickets = verifiedPool.getForEpoch(epoch);
|
|
288
|
+
const newBlock = await generator.nextBlockView(validatorIndex, key.bandersnatchSecret, sealPayload, timeSlot,
|
|
289
|
+
// VerifiedTicket has the same `{ ticket, id }` shape the generator expects.
|
|
290
|
+
[...currentEpochTickets]);
|
|
341
291
|
counter += 1;
|
|
342
292
|
lastGeneratedSlot = timeSlot;
|
|
343
293
|
logger.trace `Sending block ${counter}`;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Epoch } from "#@typeberry/block";
|
|
2
|
+
import type { SignedTicket } from "#@typeberry/block/tickets.js";
|
|
3
|
+
import type { ChainSpec } from "#@typeberry/config";
|
|
4
|
+
import type { BandernsatchWasm } from "#@typeberry/safrole/bandersnatch-wasm.js";
|
|
5
|
+
import type { State } from "#@typeberry/state";
|
|
6
|
+
import { type TicketValidator, type ValidatedTicket, ValidationError, type VerifiedTicketPool } from "#@typeberry/ticket-pool";
|
|
7
|
+
import { Result } from "#@typeberry/utils";
|
|
8
|
+
/**
|
|
9
|
+
* Real {@link TicketValidator} implementation that verifies a ticket against the ring
|
|
10
|
+
* commitment and current epoch entropy using bandersnatch, then stores the verified
|
|
11
|
+
* ticket (with its computed id) into the supplied {@link VerifiedTicketPool}.
|
|
12
|
+
*
|
|
13
|
+
* `getState` is a thunk because state advances continuously while validation is in
|
|
14
|
+
* flight; we want the latest available state for each call.
|
|
15
|
+
*/
|
|
16
|
+
export declare class BandersnatchTicketValidator implements TicketValidator {
|
|
17
|
+
private readonly bandersnatch;
|
|
18
|
+
private readonly chainSpec;
|
|
19
|
+
private readonly pool;
|
|
20
|
+
private readonly getState;
|
|
21
|
+
constructor(bandersnatch: BandernsatchWasm, chainSpec: ChainSpec, pool: VerifiedTicketPool, getState: () => State | null);
|
|
22
|
+
validate(epochIndex: Epoch, ticket: SignedTicket): Promise<Result<ValidatedTicket, ValidationError>>;
|
|
23
|
+
/**
|
|
24
|
+
* Returns the correct tickets entropy for verification given the current state.
|
|
25
|
+
*
|
|
26
|
+
* When `state` is from epoch E-1 (i.e. we haven't produced epoch E's first block yet),
|
|
27
|
+
* the ticket entropy for epoch E is at index 1 (not yet shifted). After the epoch
|
|
28
|
+
* transition it moves to index 2.
|
|
29
|
+
*/
|
|
30
|
+
private getTicketEntropy;
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=ticket-validator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ticket-validator.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/ticket-validator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,eAAe,EAEf,KAAK,kBAAkB,EACxB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAI1C;;;;;;;GAOG;AACH,qBAAa,2BAA4B,YAAW,eAAe;IAE/D,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAHR,YAAY,EAAE,gBAAgB,EAC9B,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,kBAAkB,EACxB,QAAQ,EAAE,MAAM,KAAK,GAAG,IAAI;IAGzC,QAAQ,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IA+B1G;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;CAIzB"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { Logger } from "#@typeberry/logger";
|
|
2
|
+
import bandersnatchVrf from "#@typeberry/safrole/bandersnatch-vrf.js";
|
|
3
|
+
import { ValidationError, } from "#@typeberry/ticket-pool";
|
|
4
|
+
import { Result } from "#@typeberry/utils";
|
|
5
|
+
const logger = Logger.new(import.meta.filename, "ticket-validator");
|
|
6
|
+
/**
|
|
7
|
+
* Real {@link TicketValidator} implementation that verifies a ticket against the ring
|
|
8
|
+
* commitment and current epoch entropy using bandersnatch, then stores the verified
|
|
9
|
+
* ticket (with its computed id) into the supplied {@link VerifiedTicketPool}.
|
|
10
|
+
*
|
|
11
|
+
* `getState` is a thunk because state advances continuously while validation is in
|
|
12
|
+
* flight; we want the latest available state for each call.
|
|
13
|
+
*/
|
|
14
|
+
export class BandersnatchTicketValidator {
|
|
15
|
+
bandersnatch;
|
|
16
|
+
chainSpec;
|
|
17
|
+
pool;
|
|
18
|
+
getState;
|
|
19
|
+
constructor(bandersnatch, chainSpec, pool, getState) {
|
|
20
|
+
this.bandersnatch = bandersnatch;
|
|
21
|
+
this.chainSpec = chainSpec;
|
|
22
|
+
this.pool = pool;
|
|
23
|
+
this.getState = getState;
|
|
24
|
+
}
|
|
25
|
+
async validate(epochIndex, ticket) {
|
|
26
|
+
const state = this.getState();
|
|
27
|
+
if (state === null) {
|
|
28
|
+
return Result.error(ValidationError.ValidatorUnavailable, () => "no state available");
|
|
29
|
+
}
|
|
30
|
+
const entropy = this.getTicketEntropy(epochIndex, state);
|
|
31
|
+
// Batch verifier: a single `isValid` covers the whole batch and `tickets` holds the
|
|
32
|
+
// computed id per input ticket. We only ever pass one ticket here.
|
|
33
|
+
const { isValid, tickets } = await bandersnatchVrf.verifyTickets(this.bandersnatch, state.designatedValidatorData.length, state.epochRoot, [ticket], entropy);
|
|
34
|
+
if (tickets.length !== 1) {
|
|
35
|
+
logger.error `verifyTickets returned ${tickets.length} results for 1 ticket`;
|
|
36
|
+
return Result.error(ValidationError.ValidatorUnavailable, () => "verifier returned unexpected result count");
|
|
37
|
+
}
|
|
38
|
+
if (!isValid) {
|
|
39
|
+
return Result.error(ValidationError.InvalidProof, () => "bandersnatch proof rejected");
|
|
40
|
+
}
|
|
41
|
+
const verified = { ticket, id: tickets[0] };
|
|
42
|
+
this.pool.add(epochIndex, [verified]);
|
|
43
|
+
return Result.ok({ id: tickets[0] });
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Returns the correct tickets entropy for verification given the current state.
|
|
47
|
+
*
|
|
48
|
+
* When `state` is from epoch E-1 (i.e. we haven't produced epoch E's first block yet),
|
|
49
|
+
* the ticket entropy for epoch E is at index 1 (not yet shifted). After the epoch
|
|
50
|
+
* transition it moves to index 2.
|
|
51
|
+
*/
|
|
52
|
+
getTicketEntropy(epochIndex, state) {
|
|
53
|
+
const stateEpoch = Math.floor(state.timeslot / this.chainSpec.epochLength);
|
|
54
|
+
return epochIndex > stateEpoch ? state.entropy[1] : state.entropy[2];
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -21,6 +21,16 @@ export declare const protocol: import("@typeberry/workers-api").LousyProtocol<{
|
|
|
21
21
|
}>>;
|
|
22
22
|
response: import("@typeberry/codec").Descriptor<void, void>;
|
|
23
23
|
};
|
|
24
|
+
replaceTicketPool: {
|
|
25
|
+
request: import("@typeberry/codec").Descriptor<TicketsMessage, import("@typeberry/codec").ViewOf<TicketsMessage, {
|
|
26
|
+
epochIndex: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"Epoch">, import("@typeberry/bytes").Bytes<4>>;
|
|
27
|
+
tickets: import("@typeberry/codec").Descriptor<import("@typeberry/block").SignedTicket[], import("@typeberry/codec").SequenceView<import("@typeberry/block").SignedTicket, import("@typeberry/codec").ViewOf<import("@typeberry/block").SignedTicket, {
|
|
28
|
+
attempt: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<1> & import("@typeberry/utils").WithOpaque<"TicketAttempt[u8]">, import("@typeberry/numbers").U32>;
|
|
29
|
+
signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<784> & import("@typeberry/utils").WithOpaque<"BandersnatchRingSignature">, import("@typeberry/bytes").Bytes<784>>;
|
|
30
|
+
}>>>;
|
|
31
|
+
}>>;
|
|
32
|
+
response: import("@typeberry/codec").Descriptor<void, void>;
|
|
33
|
+
};
|
|
24
34
|
}, {
|
|
25
35
|
receivedTickets: {
|
|
26
36
|
request: import("@typeberry/codec").Descriptor<ReceivedTicketMessage, import("@typeberry/codec").ViewOf<ReceivedTicketMessage, {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/comms-authorship-network/protocol.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,GAAG,EAAkB,KAAK,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE7E;;;GAGG;AACH,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAE5D;;;;GAIG;AACH,eAAO,MAAM,QAAQ
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/comms-authorship-network/protocol.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,GAAG,EAAkB,KAAK,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE7E;;;GAGG;AACH,eAAO,MAAM,uBAAuB,uBAAuB,CAAC;AAE5D;;;;GAIG;AACH,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwBnB,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;AACnD,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC"}
|
|
@@ -12,12 +12,19 @@ export const AUTHORSHIP_NETWORK_PORT = "authorship-network";
|
|
|
12
12
|
* This bypasses the main thread for ticket distribution, reducing latency.
|
|
13
13
|
*/
|
|
14
14
|
export const protocol = createProtocol("authorship-network", {
|
|
15
|
-
// Messages from block-authorship to jam-network
|
|
15
|
+
// Messages from block-authorship to jam-network.
|
|
16
16
|
toWorker: {
|
|
17
|
+
// Newly generated own tickets; networking should add them to its redistribution pool.
|
|
17
18
|
tickets: {
|
|
18
19
|
request: TicketsMessage.Codec,
|
|
19
20
|
response: codec.nothing,
|
|
20
21
|
},
|
|
22
|
+
// Authoritative pool snapshot for the given epoch; networking replaces its local
|
|
23
|
+
// pool with these tickets (one-way, source of truth lives in block-authorship).
|
|
24
|
+
replaceTicketPool: {
|
|
25
|
+
request: TicketsMessage.Codec,
|
|
26
|
+
response: codec.nothing,
|
|
27
|
+
},
|
|
21
28
|
},
|
|
22
29
|
// Messages from jam-network to block-authorship (one ticket per relay).
|
|
23
30
|
// Response indicates whether the ticket passed validation — used by jam-network
|
|
@@ -8,8 +8,8 @@ import type { TransitionHasher } from "#@typeberry/transition";
|
|
|
8
8
|
import { BlockVerifierError } from "#@typeberry/transition/block-verifier.js";
|
|
9
9
|
import { type StfError } from "#@typeberry/transition/chain-stf.js";
|
|
10
10
|
import { Result, type TaggedError } from "#@typeberry/utils";
|
|
11
|
-
import { type ImporterEventsListener } from "./events-logger.js";
|
|
12
11
|
import type { Finalizer } from "./finality.js";
|
|
12
|
+
import { type ImporterEventsListener } from "./stats.js";
|
|
13
13
|
export declare enum ImporterErrorKind {
|
|
14
14
|
Verifier = 0,
|
|
15
15
|
Stf = 1,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAiB,MAAM,kBAAkB,CAAC;AACvH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACxF,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAiB,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AAC5F,OAAO,EAA0B,KAAK,QAAQ,EAAE,MAAM,oCAAoC,CAAC;AAC3F,OAAO,EAA6B,MAAM,EAAkB,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvG,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAiB,MAAM,kBAAkB,CAAC;AACvH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACxF,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAiB,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AAC5F,OAAO,EAA0B,KAAK,QAAQ,EAAE,MAAM,oCAAoC,CAAC;AAC3F,OAAO,EAA6B,MAAM,EAAkB,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,OAAO,EAAE,KAAK,sBAAsB,EAAiB,MAAM,YAAY,CAAC;AAExE,oBAAY,iBAAiB;IAC3B,QAAQ,IAAI;IACZ,GAAG,IAAI;IACP,MAAM,IAAI;CACX;AAED,MAAM,MAAM,aAAa,GACrB,WAAW,CAAC,iBAAiB,CAAC,QAAQ,EAAE,kBAAkB,CAAC,GAC3D,WAAW,CAAC,iBAAiB,CAAC,GAAG,EAAE,QAAQ,CAAC,GAC5C,WAAW,CAAC,iBAAiB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;AAO5D,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,mDAAmD;AACnD,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,SAAS,CAAC;IAChB,GAAG,EAAE,UAAU,CAAC;IAChB,MAAM,EAAE,gBAAgB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,CAAC,EAAE,sBAAsB,CAAC;CACjC,CAAC;AAQF,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgB;IACzC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAU;IAG9B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAEhD,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2C;IAEnE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;IAC3D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;IAEhD;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,GAAG,QAAQ;IASzC,OAAO;IAyBP,6DAA6D;IAChD,mBAAmB;IAQnB,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IAQzF,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,aAAa,CAAC,CAAC;YA2B9F,mBAAmB;IA6FjC,oBAAoB;IAMpB,gBAAgB;IAIhB,eAAe,CAAC,UAAU,EAAE,UAAU;IAKhC,KAAK;CAIZ"}
|
|
@@ -3,8 +3,8 @@ import { WithHash } from "#@typeberry/hash";
|
|
|
3
3
|
import { BlockVerifier, BlockVerifierError } from "#@typeberry/transition/block-verifier.js";
|
|
4
4
|
import { DbHeaderChain, OnChain } from "#@typeberry/transition/chain-stf.js";
|
|
5
5
|
import { measure, Result, resultToString } from "#@typeberry/utils";
|
|
6
|
-
import { ImporterStats } from "./events-logger.js";
|
|
7
6
|
import * as metrics from "./metrics.js";
|
|
7
|
+
import { ImporterStats } from "./stats.js";
|
|
8
8
|
export var ImporterErrorKind;
|
|
9
9
|
(function (ImporterErrorKind) {
|
|
10
10
|
ImporterErrorKind[ImporterErrorKind["Verifier"] = 0] = "Verifier";
|
|
@@ -52,7 +52,7 @@ export class Importer {
|
|
|
52
52
|
this.blocks = args.blocks;
|
|
53
53
|
this.states = args.states;
|
|
54
54
|
this.options = args.options ?? {};
|
|
55
|
-
this.events = args.events ?? ImporterStats.new(args.logger);
|
|
55
|
+
this.events = args.events ?? ImporterStats.new(args.logger, () => this.states.diskSizeInBytes?.() ?? null);
|
|
56
56
|
this.metrics = metrics.createMetrics();
|
|
57
57
|
this.verifier = BlockVerifier.new(args.hasher, args.blocks);
|
|
58
58
|
this.stf = OnChain.assemble({
|
|
@@ -2,6 +2,8 @@ import type { HeaderHash, TimeSlot } from "#@typeberry/block";
|
|
|
2
2
|
import type { LeafDb } from "#@typeberry/database";
|
|
3
3
|
import type { Logger } from "#@typeberry/logger";
|
|
4
4
|
import type { SerializedState } from "#@typeberry/state-merkleization";
|
|
5
|
+
/** Reports the current on-disk database size in bytes, or `null` when unknown. */
|
|
6
|
+
export type DbSizeProvider = () => number | null;
|
|
5
7
|
/** Events happening during block imports. */
|
|
6
8
|
export interface ImporterEventsListener {
|
|
7
9
|
/**
|
|
@@ -16,18 +18,21 @@ export interface ImporterEventsListener {
|
|
|
16
18
|
}
|
|
17
19
|
export declare class ImporterStats implements ImporterEventsListener {
|
|
18
20
|
private readonly logger;
|
|
21
|
+
/** Reports the current on-disk database size in bytes, or `null` if unknown. */
|
|
22
|
+
private readonly dbSizeInBytes;
|
|
19
23
|
/** How often we are going to print the stats (i.e. every `maxBlocks` blocks) */
|
|
20
24
|
private readonly maxBlocks;
|
|
21
25
|
/** Alternatively print stats when we reach `${maxTimeMs}` of total block execution. */
|
|
22
26
|
private readonly maxTimeMs;
|
|
23
27
|
private readonly memory;
|
|
28
|
+
private showDiskStats;
|
|
24
29
|
private totalTimePrev;
|
|
25
30
|
private totalTime;
|
|
26
31
|
private totalBlocksPrev;
|
|
27
32
|
private totalBlocks;
|
|
28
|
-
static new(logger: Logger): ImporterStats;
|
|
33
|
+
static new(logger: Logger, dbSizeInBytes?: DbSizeProvider): ImporterStats;
|
|
29
34
|
private constructor();
|
|
30
35
|
onStart(currentBestHeaderHash: HeaderHash, currentBestState: SerializedState<LeafDb>): void;
|
|
31
36
|
onBlockImportingStarted(timeSlot: TimeSlot): (isOk: boolean) => number;
|
|
32
37
|
}
|
|
33
|
-
//# sourceMappingURL=
|
|
38
|
+
//# sourceMappingURL=stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,kFAAkF;AAClF,MAAM,MAAM,cAAc,GAAG,MAAM,MAAM,GAAG,IAAI,CAAC;AAWjD,6CAA6C;AAC7C,MAAM,WAAW,sBAAsB;IACrC;;;;;OAKG;IACH,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,CAAC;IAEvE,qCAAqC;IACrC,OAAO,CAAC,qBAAqB,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;CAC7F;AAED,qBAAa,aAAc,YAAW,sBAAsB;IAaxD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,uFAAuF;IACvF,OAAO,CAAC,QAAQ,CAAC,SAAS;IAlB5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,WAAW,CAAK;IAExB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,GAAE,cAA2B;IAIrE,OAAO;IAUP,OAAO,CAAC,qBAAqB,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,CAAC,MAAM,CAAC;IAIpF,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,IAGhC,MAAM,OAAO;CAiCxB"}
|
|
@@ -1,22 +1,35 @@
|
|
|
1
1
|
import { memoryTracker, now } from "#@typeberry/utils";
|
|
2
|
+
/** Format a database size for the stats line, e.g. ` db=12.34GB`. Empty when unknown. */
|
|
3
|
+
function formatDbSize(bytes) {
|
|
4
|
+
if (bytes === null) {
|
|
5
|
+
return "";
|
|
6
|
+
}
|
|
7
|
+
const mb = bytes / (1024 * 1024);
|
|
8
|
+
return mb >= 1024 ? ` db=${(mb / 1024).toFixed(2)}GB` : ` db=${mb.toFixed(1)}MB`;
|
|
9
|
+
}
|
|
2
10
|
export class ImporterStats {
|
|
3
11
|
logger;
|
|
12
|
+
dbSizeInBytes;
|
|
4
13
|
maxBlocks;
|
|
5
14
|
maxTimeMs;
|
|
6
15
|
memory = memoryTracker();
|
|
16
|
+
showDiskStats = true;
|
|
7
17
|
totalTimePrev = 0;
|
|
8
18
|
totalTime = 0;
|
|
9
19
|
totalBlocksPrev = 0;
|
|
10
20
|
totalBlocks = 0;
|
|
11
|
-
static new(logger) {
|
|
12
|
-
return new ImporterStats(logger);
|
|
21
|
+
static new(logger, dbSizeInBytes = () => null) {
|
|
22
|
+
return new ImporterStats(logger, dbSizeInBytes);
|
|
13
23
|
}
|
|
14
24
|
constructor(logger,
|
|
25
|
+
/** Reports the current on-disk database size in bytes, or `null` if unknown. */
|
|
26
|
+
dbSizeInBytes = () => null,
|
|
15
27
|
/** How often we are going to print the stats (i.e. every `maxBlocks` blocks) */
|
|
16
28
|
maxBlocks = 100,
|
|
17
29
|
/** Alternatively print stats when we reach `${maxTimeMs}` of total block execution. */
|
|
18
30
|
maxTimeMs = 5000) {
|
|
19
31
|
this.logger = logger;
|
|
32
|
+
this.dbSizeInBytes = dbSizeInBytes;
|
|
20
33
|
this.maxBlocks = maxBlocks;
|
|
21
34
|
this.maxTimeMs = maxTimeMs;
|
|
22
35
|
}
|
|
@@ -32,6 +45,12 @@ export class ImporterStats {
|
|
|
32
45
|
this.totalTime += duration;
|
|
33
46
|
this.totalBlocks += 1;
|
|
34
47
|
if (this.totalBlocks >= this.maxBlocks || this.totalTime >= this.maxTimeMs) {
|
|
48
|
+
// disk data (every second output)
|
|
49
|
+
if (this.showDiskStats) {
|
|
50
|
+
this.logger.info `💾 disk at #${timeSlot}: ${formatDbSize(this.dbSizeInBytes())}`;
|
|
51
|
+
}
|
|
52
|
+
this.showDiskStats = !this.showDiskStats;
|
|
53
|
+
// memory
|
|
35
54
|
this.logger.info `📊 mem at #${timeSlot}: ${this.memory()}`;
|
|
36
55
|
// compute block statistics (rolling window of last two rounds)
|
|
37
56
|
const importedBlocks = this.totalBlocks + this.totalBlocksPrev;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/jam-network/main.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/jam-network/main.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qCAAqC,CAAC;AAO3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAI1E;;;;;;GAMG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,YAAY,CAAC,gBAAgB,CAAC,EACtC,KAAK,EAAE,kBAAkB,EACzB,eAAe,EAAE,eAAe,iBAwEjC"}
|
|
@@ -2,6 +2,8 @@ import { parseBootnode } from "#@typeberry/config-node";
|
|
|
2
2
|
import { ed25519, initWasm } from "#@typeberry/crypto";
|
|
3
3
|
import { setup } from "#@typeberry/jamnp-s";
|
|
4
4
|
import { Logger } from "#@typeberry/logger";
|
|
5
|
+
import { ValidationError } from "#@typeberry/ticket-pool";
|
|
6
|
+
import { Result } from "#@typeberry/utils";
|
|
5
7
|
const logger = Logger.new(import.meta.filename, "net");
|
|
6
8
|
/**
|
|
7
9
|
* JAM networking worker.
|
|
@@ -38,11 +40,25 @@ export async function main(config, comms, authorshipComms) {
|
|
|
38
40
|
network.ticketTask.addTicket(epochIndex, ticket);
|
|
39
41
|
}
|
|
40
42
|
});
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
// Authorship pushes the authoritative ticket pool on epoch boundaries; networking
|
|
44
|
+
// replaces its redistribution pool wholesale so the two sides cannot drift.
|
|
45
|
+
authorshipComms.setOnReplaceTicketPool(async ({ epochIndex, tickets }) => {
|
|
46
|
+
logger.log `Replacing redistribution pool from block-authorship for epoch ${epochIndex} (${tickets.length} tickets)`;
|
|
47
|
+
network.ticketTask.replacePool(epochIndex, tickets);
|
|
45
48
|
});
|
|
49
|
+
// Validator that hands a received ticket to block-authorship over IPC and waits
|
|
50
|
+
// for an accept/reject decision. The wire protocol stays a simple bool; the
|
|
51
|
+
// computed id stays inside authorship (it owns the verified pool).
|
|
52
|
+
const ipcValidator = {
|
|
53
|
+
validate: async (epochIndex, ticket) => {
|
|
54
|
+
const ok = await authorshipComms.sendReceivedTickets({ epochIndex, ticket });
|
|
55
|
+
if (!ok) {
|
|
56
|
+
return Result.error(ValidationError.InvalidProof, () => "authorship rejected the ticket");
|
|
57
|
+
}
|
|
58
|
+
return Result.ok({ id: null });
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
network.ticketTask.setTicketValidator(ipcValidator);
|
|
46
62
|
await network.network.start();
|
|
47
63
|
// stop the network when the worker is finishing.
|
|
48
64
|
await waitForFinish;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"events-logger.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/events-logger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,6CAA6C;AAC7C,MAAM,WAAW,sBAAsB;IACrC;;;;;OAKG;IACH,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,CAAC;IAEvE,qCAAqC;IACrC,OAAO,CAAC,qBAAqB,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;CAC7F;AAED,qBAAa,aAAc,YAAW,sBAAsB;IAYxD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,uFAAuF;IACvF,OAAO,CAAC,QAAQ,CAAC,SAAS;IAf5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,WAAW,CAAK;IAExB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM;IAIzB,OAAO;IAQP,OAAO,CAAC,qBAAqB,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,CAAC,MAAM,CAAC;IAIpF,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,IAGhC,MAAM,OAAO;CA0BxB"}
|