@typeberry/lib 0.6.0 → 0.6.1-563c82d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/core/pvm-interpreter-ananas/index.js +1 -1
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts +7 -0
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map +1 -1
- package/packages/jam/jamnp-s/tasks/ticket-distribution.js +37 -4
- package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js +40 -1
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +2 -1
- package/packages/workers/api-node/config.d.ts +9 -0
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +10 -0
- package/packages/workers/api-node/config.test.d.ts +2 -0
- package/packages/workers/api-node/config.test.d.ts.map +1 -0
- package/packages/workers/api-node/config.test.js +41 -0
- package/packages/workers/api-node/host-environment.d.ts +19 -0
- package/packages/workers/api-node/host-environment.d.ts.map +1 -0
- package/packages/workers/api-node/host-environment.js +91 -0
- package/packages/workers/api-node/index.d.ts +1 -0
- package/packages/workers/api-node/index.d.ts.map +1 -1
- package/packages/workers/api-node/index.js +1 -0
- package/packages/workers/api-node/protocol.d.ts.map +1 -1
- package/packages/workers/api-node/protocol.js +7 -3
- package/packages/workers/block-authorship/generator.d.ts +22 -3
- package/packages/workers/block-authorship/generator.d.ts.map +1 -1
- package/packages/workers/block-authorship/generator.js +39 -5
- package/packages/workers/block-authorship/generator.test.js +202 -1
- package/packages/workers/block-authorship/main.d.ts.map +1 -1
- package/packages/workers/block-authorship/main.js +175 -25
- package/packages/workers/comms-authorship-network/protocol.d.ts +13 -2
- package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/protocol.js +10 -3
- package/packages/workers/comms-authorship-network/tickets-message.d.ts +14 -0
- package/packages/workers/comms-authorship-network/tickets-message.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/tickets-message.js +17 -0
- package/packages/workers/importer/finality.js +4 -4
- package/packages/workers/importer/finality.test.js +186 -168
- package/packages/workers/jam-network/main.d.ts.map +1 -1
- package/packages/workers/jam-network/main.js +5 -0
package/package.json
CHANGED
|
@@ -109,7 +109,7 @@ export class AnanasInterpreter {
|
|
|
109
109
|
const programArr = lowerBytes(program);
|
|
110
110
|
const argsArr = lowerBytes(args);
|
|
111
111
|
this.gas.initialGas = gas;
|
|
112
|
-
this.instance.resetJAM(programArr, pc, BigInt(gas), argsArr, true);
|
|
112
|
+
this.instance.resetJAM(programArr, pc, BigInt(gas), argsArr, true, false);
|
|
113
113
|
}
|
|
114
114
|
resetGeneric(program, _pc, gas) {
|
|
115
115
|
const programArr = lowerBytes(program);
|
|
@@ -29,6 +29,13 @@ export declare class TicketDistributionTask {
|
|
|
29
29
|
* Deduplicates tickets based on signature.
|
|
30
30
|
*/
|
|
31
31
|
addTicket(epochIndex: Epoch, ticket: SignedTicket): void;
|
|
32
|
+
private onTicketReceivedCallback;
|
|
33
|
+
/**
|
|
34
|
+
* Register a callback that validates a received ticket.
|
|
35
|
+
* The ticket is only added to the redistribution pool if the callback returns `true`.
|
|
36
|
+
* This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
|
|
37
|
+
*/
|
|
38
|
+
setOnTicketReceived(cb: (epochIndex: Epoch, ticket: SignedTicket) => Promise<boolean>): void;
|
|
32
39
|
private onTicketReceived;
|
|
33
40
|
}
|
|
34
41
|
//# sourceMappingURL=ticket-distribution.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAuB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAvB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAA0D;IAChF,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAkDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;
|
|
1
|
+
{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAuB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAvB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAA0D;IAChF,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAkDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAsCjD,OAAO,CAAC,wBAAwB,CAAgF;IAEhH;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC;IAIrF,OAAO,CAAC,gBAAgB;CAsBzB"}
|
|
@@ -82,8 +82,13 @@ export class TicketDistributionTask {
|
|
|
82
82
|
* Deduplicates tickets based on signature.
|
|
83
83
|
*/
|
|
84
84
|
addTicket(epochIndex, ticket) {
|
|
85
|
-
//
|
|
86
|
-
|
|
85
|
+
// Drop tickets for older epochs (can happen when a delayed validation callback completes
|
|
86
|
+
// after the epoch has already advanced — accepting it would roll back currentEpoch).
|
|
87
|
+
if (this.currentEpoch !== null && epochIndex < this.currentEpoch) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
// Epoch advanced — clear old tickets
|
|
91
|
+
if (this.currentEpoch !== null && epochIndex > this.currentEpoch) {
|
|
87
92
|
logger.log `[addTicket] Epoch changed from ${this.currentEpoch} to ${epochIndex}, clearing ${this.pendingTickets.length} old tickets`;
|
|
88
93
|
this.pendingTickets = [];
|
|
89
94
|
// Note: We don't need to clear aux data for all peers here.
|
|
@@ -107,9 +112,37 @@ export class TicketDistributionTask {
|
|
|
107
112
|
logger.info `[addTicket] Added ticket for epoch ${epochIndex}, total: ${this.pendingTickets.length}`;
|
|
108
113
|
}
|
|
109
114
|
}
|
|
115
|
+
onTicketReceivedCallback = null;
|
|
116
|
+
/**
|
|
117
|
+
* Register a callback that validates a received ticket.
|
|
118
|
+
* The ticket is only added to the redistribution pool if the callback returns `true`.
|
|
119
|
+
* This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
|
|
120
|
+
*/
|
|
121
|
+
setOnTicketReceived(cb) {
|
|
122
|
+
this.onTicketReceivedCallback = cb;
|
|
123
|
+
}
|
|
110
124
|
onTicketReceived(epochIndex, ticket) {
|
|
111
125
|
logger.trace `Received ticket for epoch ${epochIndex}, attempt ${ticket.attempt}`;
|
|
112
|
-
|
|
113
|
-
|
|
126
|
+
if (this.onTicketReceivedCallback !== null) {
|
|
127
|
+
// Validate first; only redistribute if valid to avoid spreading tampered tickets.
|
|
128
|
+
// Wrap with Promise.resolve().then() to catch both sync throws and async rejections.
|
|
129
|
+
const cb = this.onTicketReceivedCallback;
|
|
130
|
+
Promise.resolve()
|
|
131
|
+
.then(() => cb(epochIndex, ticket))
|
|
132
|
+
.then((isValid) => {
|
|
133
|
+
if (isValid) {
|
|
134
|
+
this.addTicket(epochIndex, ticket);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
logger.warn `Dropping invalid ticket for epoch ${epochIndex} (validation failed)`;
|
|
138
|
+
}
|
|
139
|
+
})
|
|
140
|
+
.catch((error) => {
|
|
141
|
+
logger.error `Error validating ticket for epoch ${epochIndex}, attempt ${ticket.attempt}: ${error}`;
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
this.addTicket(epochIndex, ticket);
|
|
146
|
+
}
|
|
114
147
|
}
|
|
115
148
|
}
|
|
@@ -208,7 +208,7 @@ describe("TicketDistributionTask", () => {
|
|
|
208
208
|
peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
|
|
209
209
|
peer1.ticketTask.maintainDistribution();
|
|
210
210
|
await tick();
|
|
211
|
-
// Self receives the ticket (via onTicketReceived -> addTicket)
|
|
211
|
+
// Self receives the ticket (via onTicketReceived -> addTicket, no callback set)
|
|
212
212
|
assert.strictEqual(self.receivedTickets.length, 1);
|
|
213
213
|
// Self should re-distribute to peer2 (and peer1, but peer1 already has it)
|
|
214
214
|
self.ticketTask.maintainDistribution();
|
|
@@ -217,4 +217,43 @@ describe("TicketDistributionTask", () => {
|
|
|
217
217
|
assert.strictEqual(peer2.receivedTickets.length, 1);
|
|
218
218
|
assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
|
|
219
219
|
});
|
|
220
|
+
it("should NOT redistribute ticket if validation callback returns false", async () => {
|
|
221
|
+
const self = await init("self");
|
|
222
|
+
const peer1 = await init("peer1");
|
|
223
|
+
const peer2 = await init("peer2");
|
|
224
|
+
self.openConnection(peer1);
|
|
225
|
+
self.openConnection(peer2);
|
|
226
|
+
await tick();
|
|
227
|
+
// Validation always rejects
|
|
228
|
+
self.ticketTask.setOnTicketReceived(async () => false);
|
|
229
|
+
const ticket = createTestTicket(0);
|
|
230
|
+
peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
|
|
231
|
+
peer1.ticketTask.maintainDistribution();
|
|
232
|
+
await tick();
|
|
233
|
+
// self.addTicket was NOT called (callback returned false), so nothing to redistribute
|
|
234
|
+
assert.strictEqual(self.receivedTickets.length, 0);
|
|
235
|
+
self.ticketTask.maintainDistribution();
|
|
236
|
+
await tick();
|
|
237
|
+
assert.strictEqual(peer2.receivedTickets.length, 0);
|
|
238
|
+
});
|
|
239
|
+
it("should redistribute ticket if validation callback returns true", async () => {
|
|
240
|
+
const self = await init("self");
|
|
241
|
+
const peer1 = await init("peer1");
|
|
242
|
+
const peer2 = await init("peer2");
|
|
243
|
+
self.openConnection(peer1);
|
|
244
|
+
self.openConnection(peer2);
|
|
245
|
+
await tick();
|
|
246
|
+
// Validation always accepts
|
|
247
|
+
self.ticketTask.setOnTicketReceived(async () => true);
|
|
248
|
+
const ticket = createTestTicket(0);
|
|
249
|
+
peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
|
|
250
|
+
peer1.ticketTask.maintainDistribution();
|
|
251
|
+
await tick();
|
|
252
|
+
// self.addTicket WAS called (callback returned true)
|
|
253
|
+
assert.strictEqual(self.receivedTickets.length, 1);
|
|
254
|
+
self.ticketTask.maintainDistribution();
|
|
255
|
+
await tick();
|
|
256
|
+
assert.strictEqual(peer2.receivedTickets.length, 1);
|
|
257
|
+
assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
|
|
258
|
+
});
|
|
220
259
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAe/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAKnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAe/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAKnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA0KlB"}
|
|
@@ -10,7 +10,7 @@ import { Listener } from "#@typeberry/listener";
|
|
|
10
10
|
import { tryAsU16, tryAsU32 } from "#@typeberry/numbers";
|
|
11
11
|
import { CURRENT_SUITE, CURRENT_VERSION, Result, version } from "#@typeberry/utils";
|
|
12
12
|
import { DirectPort, DirectWorkerConfig } from "#@typeberry/workers-api";
|
|
13
|
-
import { InMemWorkerConfig, LmdbWorkerConfig, ThreadPort } from "#@typeberry/workers-api-node";
|
|
13
|
+
import { InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort } from "#@typeberry/workers-api-node";
|
|
14
14
|
import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
|
|
15
15
|
import { initializeExtensions } from "./extensions.js";
|
|
16
16
|
import * as metrics from "./metrics.js";
|
|
@@ -24,6 +24,7 @@ export async function main(config, withRelPath, telemetry) {
|
|
|
24
24
|
logger.info `🫐 Typeberry ${version}. GP: ${CURRENT_VERSION} (${CURRENT_SUITE})`;
|
|
25
25
|
logger.info `🎸 Starting node: ${config.nodeName}.`;
|
|
26
26
|
logger.info `🖥️ PVM Backend: ${PvmBackend[config.pvmBackend]}.`;
|
|
27
|
+
logHostEnvironment(logger);
|
|
27
28
|
const chainSpec = getChainSpec(config.node.flavor);
|
|
28
29
|
const blake2b = await Blake2b.createHasher();
|
|
29
30
|
const nodeName = config.nodeName;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { MessagePort } from "node:worker_threads";
|
|
1
2
|
import { type Decode, type Encode } from "#@typeberry/codec";
|
|
2
3
|
import { ChainSpec } from "#@typeberry/config";
|
|
3
4
|
import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
|
|
@@ -37,6 +38,14 @@ export type TransferableConfig = {
|
|
|
37
38
|
dbPath: string;
|
|
38
39
|
workerPorts: [string, TransferablePort][];
|
|
39
40
|
};
|
|
41
|
+
/**
|
|
42
|
+
* Collect the transferable objects (communication ports) embedded in a config.
|
|
43
|
+
*
|
|
44
|
+
* `MessagePort`s can only be transferred, not structurally cloned, so they have to
|
|
45
|
+
* be listed in the `postMessage` transfer list. Omitting them results in a
|
|
46
|
+
* `DataCloneError`.
|
|
47
|
+
*/
|
|
48
|
+
export declare function configTransferList(config: TransferableConfig): MessagePort[];
|
|
40
49
|
/**
|
|
41
50
|
* In-memory (direct) worker using serialized state database.
|
|
42
51
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;IA3ChD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACjC;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IASP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;IA3ChD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACjC;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IASP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
|
|
@@ -57,6 +57,16 @@ export class LmdbWorkerConfig {
|
|
|
57
57
|
};
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Collect the transferable objects (communication ports) embedded in a config.
|
|
62
|
+
*
|
|
63
|
+
* `MessagePort`s can only be transferred, not structurally cloned, so they have to
|
|
64
|
+
* be listed in the `postMessage` transfer list. Omitting them results in a
|
|
65
|
+
* `DataCloneError`.
|
|
66
|
+
*/
|
|
67
|
+
export function configTransferList(config) {
|
|
68
|
+
return config.workerPorts.map(([, transferable]) => transferable.port);
|
|
69
|
+
}
|
|
60
70
|
/**
|
|
61
71
|
* In-memory (direct) worker using serialized state database.
|
|
62
72
|
*
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.test.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { MessageChannel } from "node:worker_threads";
|
|
4
|
+
import { codec } from "#@typeberry/codec";
|
|
5
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
6
|
+
import { Blake2b } from "#@typeberry/hash";
|
|
7
|
+
import { tryAsU32 } from "#@typeberry/numbers";
|
|
8
|
+
import { configTransferList, LmdbWorkerConfig } from "./config.js";
|
|
9
|
+
import { ThreadPort } from "./port.js";
|
|
10
|
+
const spec = tinyChainSpec;
|
|
11
|
+
describe("LmdbWorkerConfig transfer list", () => {
|
|
12
|
+
it("surfaces embedded worker ports so they can be transferred", async () => {
|
|
13
|
+
const blake2b = await Blake2b.createHasher();
|
|
14
|
+
const [portA, portB] = ThreadPort.pair(spec);
|
|
15
|
+
const config = LmdbWorkerConfig.new({
|
|
16
|
+
nodeName: "node",
|
|
17
|
+
chainSpec: spec,
|
|
18
|
+
workerParams: tryAsU32(7),
|
|
19
|
+
dbPath: "db",
|
|
20
|
+
blake2b,
|
|
21
|
+
ports: new Map([["authorship-network", portA]]),
|
|
22
|
+
});
|
|
23
|
+
const transferable = config.intoTransferable(codec.varU32);
|
|
24
|
+
const transferList = configTransferList(transferable);
|
|
25
|
+
// the single embedded comms port must be reported for transfer
|
|
26
|
+
assert.strictEqual(transferList.length, 1);
|
|
27
|
+
const sink = new MessageChannel();
|
|
28
|
+
try {
|
|
29
|
+
// reproduces the bug: a config carrying a port cannot be cloned without
|
|
30
|
+
// listing that port in the transfer list.
|
|
31
|
+
assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
|
|
32
|
+
// with the ports surfaced, posting succeeds.
|
|
33
|
+
assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
|
|
34
|
+
}
|
|
35
|
+
finally {
|
|
36
|
+
sink.port1.close();
|
|
37
|
+
sink.port2.close();
|
|
38
|
+
portB.close();
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
});
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Logger } from "#@typeberry/logger";
|
|
2
|
+
/**
|
|
3
|
+
* Log details about the host the node is running on (cpu, memory, platform)
|
|
4
|
+
* and the memory limits that apply to the process.
|
|
5
|
+
*
|
|
6
|
+
* Call this once from the main thread. Worker threads share the same OS process,
|
|
7
|
+
* so the host/cgroup/ulimit values are identical for them; use `logHeapLimit`
|
|
8
|
+
* to log the per-isolate V8 heap limit instead.
|
|
9
|
+
*/
|
|
10
|
+
export declare function logHostEnvironment(logger: Logger): void;
|
|
11
|
+
/**
|
|
12
|
+
* Log just this isolate's V8 heap limit, labelled with the worker name.
|
|
13
|
+
*
|
|
14
|
+
* Worker threads get their own V8 isolate, so this can differ from the main
|
|
15
|
+
* thread. Everything else (cpu, host/cgroup/ulimit memory) is process-wide and
|
|
16
|
+
* already logged by `logHostEnvironment` on the main thread.
|
|
17
|
+
*/
|
|
18
|
+
export declare function logHeapLimit(logger: Logger, workerName: string): void;
|
|
19
|
+
//# sourceMappingURL=host-environment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-environment.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/host-environment.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAIhD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAoBvD;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAErE"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import v8 from "node:v8";
|
|
4
|
+
const toGiB = (bytes) => `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
|
|
5
|
+
/**
|
|
6
|
+
* Log details about the host the node is running on (cpu, memory, platform)
|
|
7
|
+
* and the memory limits that apply to the process.
|
|
8
|
+
*
|
|
9
|
+
* Call this once from the main thread. Worker threads share the same OS process,
|
|
10
|
+
* so the host/cgroup/ulimit values are identical for them; use `logHeapLimit`
|
|
11
|
+
* to log the per-isolate V8 heap limit instead.
|
|
12
|
+
*/
|
|
13
|
+
export function logHostEnvironment(logger) {
|
|
14
|
+
const cpus = os.cpus();
|
|
15
|
+
const cpuModel = cpus[0]?.model ?? "unknown";
|
|
16
|
+
logger.info `💻 Platform: ${os.platform()}/${os.arch()}, Node ${process.version}.`;
|
|
17
|
+
logger.info `⚙️ CPU: ${cpuModel} (${cpus.length} cores, ${os.availableParallelism()} available).`;
|
|
18
|
+
logger.info `🧠 Memory: ${toGiB(os.freemem())} free / ${toGiB(os.totalmem())} total (host).`;
|
|
19
|
+
logger.info `📦 V8 heap limit: ${toGiB(v8.getHeapStatistics().heap_size_limit)}.`;
|
|
20
|
+
// On Linux the process can be capped well below host memory by the container
|
|
21
|
+
// runtime (cgroup) or an address-space ulimit. os.totalmem() reflects neither,
|
|
22
|
+
// so surface them explicitly to make OOMs diagnosable.
|
|
23
|
+
const cgroupLimit = readCgroupMemoryLimit();
|
|
24
|
+
if (cgroupLimit !== null) {
|
|
25
|
+
logger.info `🐳 cgroup memory limit: ${toGiB(cgroupLimit)}.`;
|
|
26
|
+
}
|
|
27
|
+
const addressSpaceLimit = readAddressSpaceLimit();
|
|
28
|
+
if (addressSpaceLimit !== null) {
|
|
29
|
+
logger.info `🚧 Address space limit (ulimit -v): ${toGiB(addressSpaceLimit)}.`;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Log just this isolate's V8 heap limit, labelled with the worker name.
|
|
34
|
+
*
|
|
35
|
+
* Worker threads get their own V8 isolate, so this can differ from the main
|
|
36
|
+
* thread. Everything else (cpu, host/cgroup/ulimit memory) is process-wide and
|
|
37
|
+
* already logged by `logHostEnvironment` on the main thread.
|
|
38
|
+
*/
|
|
39
|
+
export function logHeapLimit(logger, workerName) {
|
|
40
|
+
logger.info `📦 V8 heap limit (${workerName}): ${toGiB(v8.getHeapStatistics().heap_size_limit)}.`;
|
|
41
|
+
}
|
|
42
|
+
/** Read the cgroup (v2, then v1) memory limit in bytes, or null if unlimited / unavailable. */
|
|
43
|
+
function readCgroupMemoryLimit() {
|
|
44
|
+
if (os.platform() !== "linux") {
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
// cgroup v2 first, then the v1 fallback path.
|
|
48
|
+
for (const path of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
|
|
49
|
+
try {
|
|
50
|
+
const raw = fs.readFileSync(path, "utf8").trim();
|
|
51
|
+
if (raw === "max") {
|
|
52
|
+
return null; // cgroup v2 sentinel for "no limit"
|
|
53
|
+
}
|
|
54
|
+
const value = Number(raw);
|
|
55
|
+
// cgroup v1 reports a value close to 2^63 when unbounded; treat anything at or
|
|
56
|
+
// above host memory as effectively no limit.
|
|
57
|
+
if (!Number.isFinite(value) || value <= 0 || value >= os.totalmem()) {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
return value;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// file missing or unreadable, fall through to the next candidate
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
/** Read the soft RLIMIT_AS (address space, i.e. `ulimit -v`) in bytes, or null if unlimited / unavailable. */
|
|
69
|
+
function readAddressSpaceLimit() {
|
|
70
|
+
if (os.platform() !== "linux") {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
try {
|
|
74
|
+
const limits = fs.readFileSync("/proc/self/limits", "utf8");
|
|
75
|
+
const line = limits.split("\n").find((l) => l.startsWith("Max address space"));
|
|
76
|
+
if (line === undefined) {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
// Columns are: name (multi-word), soft limit, hard limit, units.
|
|
80
|
+
const soft = line.slice("Max address space".length).trim().split(/\s+/)[0];
|
|
81
|
+
if (soft === undefined || soft === "unlimited") {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const value = Number(soft);
|
|
85
|
+
return Number.isFinite(value) && value > 0 ? value : null;
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
// /proc not mounted or unreadable
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAsB,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAM5F,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAChC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAoCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAmDD"}
|
|
@@ -3,7 +3,8 @@ import { Listener } from "#@typeberry/listener";
|
|
|
3
3
|
import { Level, Logger } from "#@typeberry/logger";
|
|
4
4
|
import { assertNever } from "#@typeberry/utils";
|
|
5
5
|
import { Channel } from "#@typeberry/workers-api";
|
|
6
|
-
import { LmdbWorkerConfig } from "./config.js";
|
|
6
|
+
import { configTransferList, LmdbWorkerConfig } from "./config.js";
|
|
7
|
+
import { logHeapLimit } from "./host-environment.js";
|
|
7
8
|
import { ThreadPort } from "./port.js";
|
|
8
9
|
const logger = Logger.new(import.meta.filename, "workers");
|
|
9
10
|
/** Type of the control plane message. */
|
|
@@ -37,8 +38,10 @@ export function spawnWorker(protocol, bootstrapPath, config, paramsEncoder) {
|
|
|
37
38
|
config: config.intoTransferable(paramsEncoder),
|
|
38
39
|
};
|
|
39
40
|
logger.trace `(${protocol.name}) <-- config`;
|
|
40
|
-
// send the config down to the worker
|
|
41
|
-
worker
|
|
41
|
+
// send the config down to the worker. We need to transfer the parent
|
|
42
|
+
// communication port as well as any inter-worker ports carried in the config,
|
|
43
|
+
// otherwise structured clone fails with a `DataCloneError`.
|
|
44
|
+
worker.postMessage(msg, [msg.parentPort, ...configTransferList(msg.config)]);
|
|
42
45
|
const workerFinished = new Promise((resolve, reject) => {
|
|
43
46
|
worker.once("error", reject);
|
|
44
47
|
worker.once("exit", (exitCode) => {
|
|
@@ -65,6 +68,7 @@ export async function initWorker(protocol, paramsDecoder) {
|
|
|
65
68
|
// configure logger inside a worker thread
|
|
66
69
|
Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG);
|
|
67
70
|
logger.trace `Worker ${protocol.name} starting.`;
|
|
71
|
+
logHeapLimit(logger, protocol.name);
|
|
68
72
|
return new Promise((resolve, reject) => {
|
|
69
73
|
if (parentPort === null) {
|
|
70
74
|
throw new Error(`Unable to start ${protocol.name} worker. Not running in a worker thread!`);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { Block, type TimeSlot, type ValidatorIndex } from "#@typeberry/block";
|
|
1
|
+
import { Block, type EntropyHash, type TimeSlot, type ValidatorIndex } from "#@typeberry/block";
|
|
2
2
|
import { type BlockView } from "#@typeberry/block/block.js";
|
|
3
|
+
import type { SignedTicket } from "#@typeberry/block/tickets.js";
|
|
3
4
|
import { BytesBlob } from "#@typeberry/bytes";
|
|
4
5
|
import type { ChainSpec } from "#@typeberry/config";
|
|
5
6
|
import { type BandersnatchSecretSeed } from "#@typeberry/crypto";
|
|
@@ -38,7 +39,10 @@ export declare class Generator {
|
|
|
38
39
|
static new(args: GeneratorArgs): Generator;
|
|
39
40
|
private constructor();
|
|
40
41
|
private getLastHeaderAndState;
|
|
41
|
-
nextBlockView(validatorIndex: ValidatorIndex, bandersnatchSecret: BandersnatchSecretSeed, sealPayload: BlockSealInput, timeSlot: TimeSlot
|
|
42
|
+
nextBlockView(validatorIndex: ValidatorIndex, bandersnatchSecret: BandersnatchSecretSeed, sealPayload: BlockSealInput, timeSlot: TimeSlot, pendingTickets?: {
|
|
43
|
+
ticket: SignedTicket;
|
|
44
|
+
id: EntropyHash;
|
|
45
|
+
}[]): Promise<BlockView>;
|
|
42
46
|
/**
|
|
43
47
|
* Returns y(H_S) part of the VRF signature.
|
|
44
48
|
*
|
|
@@ -50,6 +54,21 @@ export declare class Generator {
|
|
|
50
54
|
* data (i.e. the `aux_data`) so we are able to compute it beforehand.
|
|
51
55
|
*/
|
|
52
56
|
private getEntropyHash;
|
|
53
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Selects tickets to include in the extrinsic from the pending pool.
|
|
59
|
+
*
|
|
60
|
+
* Tickets were already verified at receipt time (IDs pre-computed). This method:
|
|
61
|
+
* 1. Filters out tickets whose IDs are already in `state.ticketsAccumulator` (already processed).
|
|
62
|
+
* 2. Sorts remaining tickets by ID ascending (required by Safrole).
|
|
63
|
+
* 3. Deduplicates by ID (pool dedup is best-effort; reorgs can produce duplicates).
|
|
64
|
+
* 4. Returns at most `chainSpec.maxTicketsPerExtrinsic` tickets.
|
|
65
|
+
*
|
|
66
|
+
* Called only during the contest period (slotInEpoch < contestLength).
|
|
67
|
+
*/
|
|
68
|
+
private prepareTicketsExtrinsic;
|
|
69
|
+
nextBlock(validatorIndex: ValidatorIndex, bandersnatchSecret: BandersnatchSecretSeed, sealPayload: BlockSealInput, timeSlot: TimeSlot, pendingTickets?: {
|
|
70
|
+
ticket: SignedTicket;
|
|
71
|
+
id: EntropyHash;
|
|
72
|
+
}[]): Promise<Block>;
|
|
54
73
|
}
|
|
55
74
|
//# sourceMappingURL=generator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/generator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,
|
|
1
|
+
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/generator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,KAAK,WAAW,EAIhB,KAAK,QAAQ,EACb,KAAK,cAAc,EACpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,KAAK,SAAS,EAAa,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAS,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAoC,KAAK,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAClG,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAIvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAGhF,OAAO,EAAqB,KAAK,MAAM,EAAU,MAAM,kBAAkB,CAAC;AAM1E;;;;;;;;GAQG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AAEvD,oDAAoD;AACpD,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,gBAAgB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,QAAQ,CAAC;CAClB,CAAC;AAEF,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2C;IAEnE,SAAgB,SAAS,EAAE,SAAS,CAAC;IACrC,SAAgB,YAAY,EAAE,gBAAgB,CAAC;IAC/C,SAAgB,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;IAClD,SAAgB,OAAO,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAElC,wDAAwD;IACxD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa;IAI9B,OAAO;IAUP,OAAO,CAAC,qBAAqB;IAYvB,aAAa,CACjB,cAAc,EAAE,cAAc,EAC9B,kBAAkB,EAAE,sBAAsB,EAC1C,WAAW,EAAE,cAAc,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,GAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,EAAE,EAAE,WAAW,CAAA;KAAE,EAAO,GAC/D,OAAO,CAAC,SAAS,CAAC;IAKrB;;;;;;;;;OASG;YACW,cAAc;IAiB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,uBAAuB;IA4BzB,SAAS,CACb,cAAc,EAAE,cAAc,EAC9B,kBAAkB,EAAE,sBAAsB,EAC1C,WAAW,EAAE,cAAc,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,GAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,EAAE,EAAE,WAAW,CAAA;KAAE,EAAO;CAqGnE"}
|
|
@@ -2,6 +2,7 @@ import { Block, encodeUnsealedHeader, Header, reencodeAsView, } from "#@typeberr
|
|
|
2
2
|
import { Extrinsic } from "#@typeberry/block/block.js";
|
|
3
3
|
import { DisputesExtrinsic } from "#@typeberry/block/disputes.js";
|
|
4
4
|
import { Bytes, BytesBlob } from "#@typeberry/bytes";
|
|
5
|
+
import { HashSet } from "#@typeberry/collections/hash-set.js";
|
|
5
6
|
import { BANDERSNATCH_VRF_SIGNATURE_BYTES } from "#@typeberry/crypto";
|
|
6
7
|
import { Logger } from "#@typeberry/logger";
|
|
7
8
|
import { Safrole } from "#@typeberry/safrole";
|
|
@@ -44,8 +45,8 @@ export class Generator {
|
|
|
44
45
|
lastState,
|
|
45
46
|
};
|
|
46
47
|
}
|
|
47
|
-
async nextBlockView(validatorIndex, bandersnatchSecret, sealPayload, timeSlot) {
|
|
48
|
-
const newBlock = await this.nextBlock(validatorIndex, bandersnatchSecret, sealPayload, timeSlot);
|
|
48
|
+
async nextBlockView(validatorIndex, bandersnatchSecret, sealPayload, timeSlot, pendingTickets = []) {
|
|
49
|
+
const newBlock = await this.nextBlock(validatorIndex, bandersnatchSecret, sealPayload, timeSlot, pendingTickets);
|
|
49
50
|
return reencodeAsView(Block.Codec, newBlock, this.chainSpec);
|
|
50
51
|
}
|
|
51
52
|
/**
|
|
@@ -65,7 +66,37 @@ export class Generator {
|
|
|
65
66
|
}
|
|
66
67
|
return entropyHashResult;
|
|
67
68
|
}
|
|
68
|
-
|
|
69
|
+
/**
|
|
70
|
+
* Selects tickets to include in the extrinsic from the pending pool.
|
|
71
|
+
*
|
|
72
|
+
* Tickets were already verified at receipt time (IDs pre-computed). This method:
|
|
73
|
+
* 1. Filters out tickets whose IDs are already in `state.ticketsAccumulator` (already processed).
|
|
74
|
+
* 2. Sorts remaining tickets by ID ascending (required by Safrole).
|
|
75
|
+
* 3. Deduplicates by ID (pool dedup is best-effort; reorgs can produce duplicates).
|
|
76
|
+
* 4. Returns at most `chainSpec.maxTicketsPerExtrinsic` tickets.
|
|
77
|
+
*
|
|
78
|
+
* Called only during the contest period (slotInEpoch < contestLength).
|
|
79
|
+
*/
|
|
80
|
+
prepareTicketsExtrinsic(pendingTickets, state) {
|
|
81
|
+
if (pendingTickets.length === 0) {
|
|
82
|
+
return [];
|
|
83
|
+
}
|
|
84
|
+
// Tickets are already verified at receipt time — just filter, sort and slice.
|
|
85
|
+
// Build a set of ticket IDs already in the state accumulator for fast lookup.
|
|
86
|
+
const accumulatedIds = HashSet.from(state.ticketsAccumulator.map((t) => t.id));
|
|
87
|
+
const filtered = pendingTickets.filter(({ id }) => !accumulatedIds.has(id));
|
|
88
|
+
// Sort by ID ascending
|
|
89
|
+
filtered.sort((a, b) => a.id.compare(b.id).value);
|
|
90
|
+
// Deduplicate by ID (pool dedup is best-effort; state may produce duplicates across reorgs)
|
|
91
|
+
const deduped = [];
|
|
92
|
+
for (const item of filtered) {
|
|
93
|
+
if (deduped.length === 0 || !deduped[deduped.length - 1].id.isEqualTo(item.id)) {
|
|
94
|
+
deduped.push(item);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return deduped.slice(0, this.chainSpec.maxTicketsPerExtrinsic).map(({ ticket }) => ticket);
|
|
98
|
+
}
|
|
99
|
+
async nextBlock(validatorIndex, bandersnatchSecret, sealPayload, timeSlot, pendingTickets = []) {
|
|
69
100
|
this.metrics.recordBlockAuthoringStarted(timeSlot);
|
|
70
101
|
const startTime = now();
|
|
71
102
|
// fetch latest data from the db.
|
|
@@ -85,9 +116,12 @@ export class Generator {
|
|
|
85
116
|
// retrieve data from previous block
|
|
86
117
|
const hasher = TransitionHasher.new(this.keccakHasher, this.blake2b);
|
|
87
118
|
const stateRoot = this.states.getStateRoot(lastState);
|
|
88
|
-
|
|
119
|
+
const slotInEpoch = timeSlot % this.chainSpec.epochLength;
|
|
120
|
+
const isContestPeriod = slotInEpoch < this.chainSpec.contestLength;
|
|
121
|
+
// Include tickets only during contest period
|
|
122
|
+
const ticketsForExtrinsic = isContestPeriod ? await this.prepareTicketsExtrinsic(pendingTickets, lastState) : [];
|
|
89
123
|
const extrinsic = Extrinsic.create({
|
|
90
|
-
tickets: asOpaqueType(
|
|
124
|
+
tickets: asOpaqueType(ticketsForExtrinsic),
|
|
91
125
|
preimages: [],
|
|
92
126
|
guarantees: asOpaqueType([]),
|
|
93
127
|
assurances: asOpaqueType([]),
|