@typeberry/lib 0.6.0 → 0.6.1-3e97e3e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/package.json +2 -2
  2. package/packages/core/concurrent/parent.d.ts +3 -0
  3. package/packages/core/concurrent/parent.d.ts.map +1 -1
  4. package/packages/core/concurrent/parent.js +9 -6
  5. package/packages/core/pvm-interpreter-ananas/index.d.ts.map +1 -1
  6. package/packages/core/pvm-interpreter-ananas/index.js +4 -2
  7. package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts +7 -0
  8. package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map +1 -1
  9. package/packages/jam/jamnp-s/tasks/ticket-distribution.js +37 -4
  10. package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js +40 -1
  11. package/packages/jam/node/main.d.ts.map +1 -1
  12. package/packages/jam/node/main.js +2 -1
  13. package/packages/workers/api-node/config.d.ts +9 -0
  14. package/packages/workers/api-node/config.d.ts.map +1 -1
  15. package/packages/workers/api-node/config.js +10 -0
  16. package/packages/workers/api-node/config.test.d.ts +2 -0
  17. package/packages/workers/api-node/config.test.d.ts.map +1 -0
  18. package/packages/workers/api-node/config.test.js +41 -0
  19. package/packages/workers/api-node/host-environment.d.ts +32 -0
  20. package/packages/workers/api-node/host-environment.d.ts.map +1 -0
  21. package/packages/workers/api-node/host-environment.js +106 -0
  22. package/packages/workers/api-node/index.d.ts +1 -0
  23. package/packages/workers/api-node/index.d.ts.map +1 -1
  24. package/packages/workers/api-node/index.js +1 -0
  25. package/packages/workers/api-node/protocol.d.ts.map +1 -1
  26. package/packages/workers/api-node/protocol.js +8 -4
  27. package/packages/workers/block-authorship/generator.d.ts +22 -3
  28. package/packages/workers/block-authorship/generator.d.ts.map +1 -1
  29. package/packages/workers/block-authorship/generator.js +39 -5
  30. package/packages/workers/block-authorship/generator.test.js +202 -1
  31. package/packages/workers/block-authorship/main.d.ts.map +1 -1
  32. package/packages/workers/block-authorship/main.js +175 -25
  33. package/packages/workers/comms-authorship-network/protocol.d.ts +13 -2
  34. package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
  35. package/packages/workers/comms-authorship-network/protocol.js +10 -3
  36. package/packages/workers/comms-authorship-network/tickets-message.d.ts +14 -0
  37. package/packages/workers/comms-authorship-network/tickets-message.d.ts.map +1 -1
  38. package/packages/workers/comms-authorship-network/tickets-message.js +17 -0
  39. package/packages/workers/importer/finality.js +4 -4
  40. package/packages/workers/importer/finality.test.js +186 -168
  41. package/packages/workers/jam-network/main.d.ts.map +1 -1
  42. package/packages/workers/jam-network/main.js +5 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@typeberry/lib",
3
- "version": "0.6.0",
3
+ "version": "0.6.1-3e97e3e",
4
4
  "description": "Typeberry Library",
5
5
  "main": "./bin/lib/index.js",
6
6
  "types": "./bin/lib/index.d.ts",
@@ -264,7 +264,7 @@
264
264
  "#@typeberry/jam-network/*": "./packages/workers/jam-network/*"
265
265
  },
266
266
  "dependencies": {
267
- "@fluffylabs/anan-as": "^1.3.0",
267
+ "@fluffylabs/anan-as": "^1.4.0",
268
268
  "@noble/ed25519": "2.2.3",
269
269
  "hash-wasm": "4.12.0",
270
270
  "@typeberry/native": "0.2.0-74dd7d7",
@@ -10,12 +10,15 @@ export type ExecutorOptions = {
10
10
  * when there is too many tasks pending in the queue.
11
11
  */
12
12
  maxWorkers: number;
13
+ /** Worker heap size limit (defaults to 2048 MB) */
14
+ maxHeapSize?: number;
13
15
  };
14
16
  /** Execution pool manager. */
15
17
  export declare class Executor<TParams extends WithTransferList, TResult> implements IExecutor<TParams, TResult> {
16
18
  private readonly workers;
17
19
  private readonly maxWorkers;
18
20
  private readonly workerPath;
21
+ private readonly maxHeapSize?;
19
22
  /** Initialize a new concurrent executor given a path to the worker. */
20
23
  static initialize<XParams extends WithTransferList, XResult extends WithTransferList>(workerPath: URL, options: ExecutorOptions): Promise<Executor<XParams, XResult>>;
21
24
  private readonly freeWorkerIndices;
@@ -1 +1 @@
1
- {"version":3,"file":"parent.d.ts","sourceRoot":"","sources":["../../../../../packages/core/concurrent/parent.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAyB,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMxF,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG;IAC5B,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8BAA8B;AAC9B,qBAAa,QAAQ,CAAC,OAAO,SAAS,gBAAgB,EAAE,OAAO,CAAE,YAAW,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC;IAsBnG,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAvB7B,uEAAuE;WAC1D,UAAU,CAAC,OAAO,SAAS,gBAAgB,EAAE,OAAO,SAAS,gBAAgB,EACxF,UAAU,EAAE,GAAG,EACf,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAWtC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAgB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,oBAAoB,CAAS;IAErC,OAAO;IAWP,0CAA0C;IACpC,aAAa,CAAC,SAAS,GAAE,MAAM,IAAe;IAiBpD,oDAAoD;IAC9C,OAAO;IASb,4CAA4C;IACtC,GAAG,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAgB5C,kDAAkD;IAClD,OAAO,CAAC,yBAAyB;CA6BlC"}
1
+ {"version":3,"file":"parent.d.ts","sourceRoot":"","sources":["../../../../../packages/core/concurrent/parent.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAyB,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAOxF,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG;IAC5B,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,8BAA8B;AAC9B,qBAAa,QAAQ,CAAC,OAAO,SAAS,gBAAgB,EAAE,OAAO,CAAE,YAAW,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC;IAsBnG,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;IAxB/B,uEAAuE;WAC1D,UAAU,CAAC,OAAO,SAAS,gBAAgB,EAAE,OAAO,SAAS,gBAAgB,EACxF,UAAU,EAAE,GAAG,EACf,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAWtC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAgB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,oBAAoB,CAAS;IAErC,OAAO;IAYP,0CAA0C;IACpC,aAAa,CAAC,SAAS,GAAE,MAAM,IAAe;IAiBpD,oDAAoD;IAC9C,OAAO;IASb,4CAA4C;IACtC,GAAG,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAgB5C,kDAAkD;IAClD,OAAO,CAAC,yBAAyB;CA6BlC"}
@@ -3,30 +3,33 @@ import { check } from "#@typeberry/utils";
3
3
  // Amount of tasks in the queue that will trigger creation of new worker thread.
4
4
  // NOTE this might need to be configurable in the future.
5
5
  const QUEUE_SIZE_WORKER_THRESHOLD = 5;
6
+ const DEFAULT_MAX_HEAP_SIZE_MB = 2048;
6
7
  /** Execution pool manager. */
7
8
  export class Executor {
8
9
  workers;
9
10
  maxWorkers;
10
11
  workerPath;
12
+ maxHeapSize;
11
13
  /** Initialize a new concurrent executor given a path to the worker. */
12
14
  static async initialize(workerPath, options) {
13
15
  check `${options.maxWorkers > 0} Max workers has to be positive.`;
14
16
  check `${options.minWorkers <= options.maxWorkers} Min workers must be less than or equal to max workers (min=${options.minWorkers}, max=${options.maxWorkers}).`;
15
17
  const workers = [];
16
18
  for (let i = 0; i < options.minWorkers; i++) {
17
- workers.push(await initWorker(workerPath));
19
+ workers.push(await initWorker(workerPath, options.maxHeapSize));
18
20
  }
19
- return new Executor(workers, options.maxWorkers, workerPath);
21
+ return new Executor(workers, options.maxWorkers, workerPath, options.maxHeapSize);
20
22
  }
21
23
  // keeps track of the indices of worker threads that are currently free and available to execute tasks
22
24
  freeWorkerIndices = [];
23
25
  taskQueue = [];
24
26
  isDestroyed = false;
25
27
  isWorkerInitializing = false;
26
- constructor(workers, maxWorkers, workerPath) {
28
+ constructor(workers, maxWorkers, workerPath, maxHeapSize) {
27
29
  this.workers = workers;
28
30
  this.maxWorkers = maxWorkers;
29
31
  this.workerPath = workerPath;
32
+ this.maxHeapSize = maxHeapSize;
30
33
  // intial free workers.
31
34
  for (let i = 0; i < workers.length; i++) {
32
35
  this.freeWorkerIndices.push(i);
@@ -43,7 +46,7 @@ export class Executor {
43
46
  return;
44
47
  }
45
48
  this.isWorkerInitializing = true;
46
- this.workers.push(await initWorker(this.workerPath));
49
+ this.workers.push(await initWorker(this.workerPath, this.maxHeapSize));
47
50
  this.freeWorkerIndices.push(this.workers.length - 1);
48
51
  this.isWorkerInitializing = false;
49
52
  onSuccess();
@@ -101,10 +104,10 @@ export class Executor {
101
104
  });
102
105
  }
103
106
  }
104
- async function initWorker(workerPath) {
107
+ async function initWorker(workerPath, maxOldGenerationSizeMb = DEFAULT_MAX_HEAP_SIZE_MB) {
105
108
  // create a worker and initialize communication channel
106
109
  const { port1, port2 } = new MessageChannel();
107
- const workerThread = new Worker(workerPath, {});
110
+ const workerThread = new Worker(workerPath, { resourceLimits: { maxOldGenerationSizeMb } });
108
111
  workerThread.postMessage(port1, [port1]);
109
112
  // // wait for the worker to start
110
113
  await new Promise((resolve, reject) => {
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/core/pvm-interpreter-ananas/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AACtE,OAAO,EAAY,KAAK,GAAG,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,KAAK,GAAG,EAER,KAAK,WAAW,EAChB,KAAK,OAAO,EACZ,KAAK,eAAe,EACpB,KAAK,UAAU,EAEf,KAAK,SAAS,EAEd,MAAM,EAGP,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAS,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAErD,KAAK,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC;AAKtD,cAAM,eAAgB,YAAW,UAAU;IAKrB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ7C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAI3B,OAAO;IAEP,aAAa,IAAI,UAAU;IAI3B,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAOtC,SAAS,IAAI,cAAc;CAI5B;AAED,cAAM,YAAa,YAAW,OAAO;IAKf,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ7C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAI3B,OAAO;IAEP,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC;IAO7D,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,GAAG,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC;CAW9D;AAED,cAAM,gBAAiB,YAAW,WAAW;IAOvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN7C,UAAU,EAAE,GAAG,CAAgB;IAE/B,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAI3B,OAAO;IAEP,GAAG,IAAI,GAAG;IAIV,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI;IAIjB,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,OAAO;IAUpB,IAAI,IAAI,GAAG;CASZ;AAED,qBAAa,iBAAkB,YAAW,eAAe;IAKnC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ7C,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC;IAE/B,OAAO;WAMM,GAAG;IAWhB,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI;IAO3E,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI;IAS9D,QAAQ,IAAI,OAAO;IAInB,UAAU,IAAI,IAAI;IAMlB,SAAS,IAAI,MAAM;IASnB,KAAK,IAAI,MAAM;IAIf,YAAY,IAAI,GAAG,GAAG,IAAI;CAG3B"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/core/pvm-interpreter-ananas/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AACtE,OAAO,EAAY,KAAK,GAAG,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,KAAK,GAAG,EAER,KAAK,WAAW,EAChB,KAAK,OAAO,EACZ,KAAK,eAAe,EACpB,KAAK,UAAU,EAEf,KAAK,SAAS,EAEd,MAAM,EAGP,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAS,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAErD,KAAK,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC;AAKtD,cAAM,eAAgB,YAAW,UAAU;IAKrB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ7C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAI3B,OAAO;IAEP,aAAa,IAAI,UAAU;IAI3B,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAOtC,SAAS,IAAI,cAAc;CAI5B;AAED,cAAM,YAAa,YAAW,OAAO;IAKf,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ7C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAI3B,OAAO;IAEP,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC;IAO7D,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,GAAG,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC;CAW9D;AAED,cAAM,gBAAiB,YAAW,WAAW;IAOvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN7C,UAAU,EAAE,GAAG,CAAgB;IAE/B,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAI3B,OAAO;IAEP,GAAG,IAAI,GAAG;IAIV,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI;IAIjB,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,OAAO;IAUpB,IAAI,IAAI,GAAG;CASZ;AAKD,qBAAa,iBAAkB,YAAW,eAAe;IAKnC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ7C,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC;IAE/B,OAAO;WAMM,GAAG;IAWhB,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI;IAO3E,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI;IAkB9D,QAAQ,IAAI,OAAO;IAInB,UAAU,IAAI,IAAI;IAMlB,SAAS,IAAI,MAAM;IASnB,KAAK,IAAI,MAAM;IAIf,YAAY,IAAI,GAAG,GAAG,IAAI;CAG3B"}
@@ -84,6 +84,8 @@ class AnanasGasCounter {
84
84
  return tryAsBigGas(gasConsumed);
85
85
  }
86
86
  }
87
+ const USE_BLOCK_GAS = false;
88
+ const PAGES_TO_PREALLOCATE = 192;
87
89
  export class AnanasInterpreter {
88
90
  instance;
89
91
  registers;
@@ -109,7 +111,7 @@ export class AnanasInterpreter {
109
111
  const programArr = lowerBytes(program);
110
112
  const argsArr = lowerBytes(args);
111
113
  this.gas.initialGas = gas;
112
- this.instance.resetJAM(programArr, pc, BigInt(gas), argsArr, true);
114
+ this.instance.resetJAM(programArr, pc, BigInt(gas), argsArr, true, USE_BLOCK_GAS, PAGES_TO_PREALLOCATE);
113
115
  }
114
116
  resetGeneric(program, _pc, gas) {
115
117
  const programArr = lowerBytes(program);
@@ -117,7 +119,7 @@ export class AnanasInterpreter {
117
119
  const pageMap = new Uint8Array();
118
120
  const chunks = new Uint8Array();
119
121
  this.gas.initialGas = gas;
120
- this.instance.resetGenericWithMemory(programArr, emptyRegisters, pageMap, chunks, BigInt(gas), false);
122
+ this.instance.resetGenericWithMemory(programArr, emptyRegisters, pageMap, chunks, BigInt(gas), false, USE_BLOCK_GAS, PAGES_TO_PREALLOCATE);
121
123
  }
122
124
  nextStep() {
123
125
  return this.instance.nextStep();
@@ -29,6 +29,13 @@ export declare class TicketDistributionTask {
29
29
  * Deduplicates tickets based on signature.
30
30
  */
31
31
  addTicket(epochIndex: Epoch, ticket: SignedTicket): void;
32
+ private onTicketReceivedCallback;
33
+ /**
34
+ * Register a callback that validates a received ticket.
35
+ * The ticket is only added to the redistribution pool if the callback returns `true`.
36
+ * This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
37
+ */
38
+ setOnTicketReceived(cb: (epochIndex: Epoch, ticket: SignedTicket) => Promise<boolean>): void;
32
39
  private onTicketReceived;
33
40
  }
34
41
  //# sourceMappingURL=ticket-distribution.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAuB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAvB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAA0D;IAChF,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAkDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAgCjD,OAAO,CAAC,gBAAgB;CAKzB"}
1
+ {"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAuB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAvB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAA0D;IAChF,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAkDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAsCjD,OAAO,CAAC,wBAAwB,CAAgF;IAEhH;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC;IAIrF,OAAO,CAAC,gBAAgB;CAsBzB"}
@@ -82,8 +82,13 @@ export class TicketDistributionTask {
82
82
  * Deduplicates tickets based on signature.
83
83
  */
84
84
  addTicket(epochIndex, ticket) {
85
- // Check if epoch changed - if so, clear old tickets
86
- if (this.currentEpoch !== null && this.currentEpoch !== epochIndex) {
85
+ // Drop tickets for older epochs (can happen when a delayed validation callback completes
86
+ // after the epoch has already advanced — accepting it would roll back currentEpoch).
87
+ if (this.currentEpoch !== null && epochIndex < this.currentEpoch) {
88
+ return;
89
+ }
90
+ // Epoch advanced — clear old tickets
91
+ if (this.currentEpoch !== null && epochIndex > this.currentEpoch) {
87
92
  logger.log `[addTicket] Epoch changed from ${this.currentEpoch} to ${epochIndex}, clearing ${this.pendingTickets.length} old tickets`;
88
93
  this.pendingTickets = [];
89
94
  // Note: We don't need to clear aux data for all peers here.
@@ -107,9 +112,37 @@ export class TicketDistributionTask {
107
112
  logger.info `[addTicket] Added ticket for epoch ${epochIndex}, total: ${this.pendingTickets.length}`;
108
113
  }
109
114
  }
115
+ onTicketReceivedCallback = null;
116
+ /**
117
+ * Register a callback that validates a received ticket.
118
+ * The ticket is only added to the redistribution pool if the callback returns `true`.
119
+ * This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
120
+ */
121
+ setOnTicketReceived(cb) {
122
+ this.onTicketReceivedCallback = cb;
123
+ }
110
124
  onTicketReceived(epochIndex, ticket) {
111
125
  logger.trace `Received ticket for epoch ${epochIndex}, attempt ${ticket.attempt}`;
112
- // Add to pending queue for potential re-distribution
113
- this.addTicket(epochIndex, ticket);
126
+ if (this.onTicketReceivedCallback !== null) {
127
+ // Validate first; only redistribute if valid to avoid spreading tampered tickets.
128
+ // Wrap with Promise.resolve().then() to catch both sync throws and async rejections.
129
+ const cb = this.onTicketReceivedCallback;
130
+ Promise.resolve()
131
+ .then(() => cb(epochIndex, ticket))
132
+ .then((isValid) => {
133
+ if (isValid) {
134
+ this.addTicket(epochIndex, ticket);
135
+ }
136
+ else {
137
+ logger.warn `Dropping invalid ticket for epoch ${epochIndex} (validation failed)`;
138
+ }
139
+ })
140
+ .catch((error) => {
141
+ logger.error `Error validating ticket for epoch ${epochIndex}, attempt ${ticket.attempt}: ${error}`;
142
+ });
143
+ }
144
+ else {
145
+ this.addTicket(epochIndex, ticket);
146
+ }
114
147
  }
115
148
  }
@@ -208,7 +208,7 @@ describe("TicketDistributionTask", () => {
208
208
  peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
209
209
  peer1.ticketTask.maintainDistribution();
210
210
  await tick();
211
- // Self receives the ticket (via onTicketReceived -> addTicket)
211
+ // Self receives the ticket (via onTicketReceived -> addTicket, no callback set)
212
212
  assert.strictEqual(self.receivedTickets.length, 1);
213
213
  // Self should re-distribute to peer2 (and peer1, but peer1 already has it)
214
214
  self.ticketTask.maintainDistribution();
@@ -217,4 +217,43 @@ describe("TicketDistributionTask", () => {
217
217
  assert.strictEqual(peer2.receivedTickets.length, 1);
218
218
  assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
219
219
  });
220
+ it("should NOT redistribute ticket if validation callback returns false", async () => {
221
+ const self = await init("self");
222
+ const peer1 = await init("peer1");
223
+ const peer2 = await init("peer2");
224
+ self.openConnection(peer1);
225
+ self.openConnection(peer2);
226
+ await tick();
227
+ // Validation always rejects
228
+ self.ticketTask.setOnTicketReceived(async () => false);
229
+ const ticket = createTestTicket(0);
230
+ peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
231
+ peer1.ticketTask.maintainDistribution();
232
+ await tick();
233
+ // self.addTicket was NOT called (callback returned false), so nothing to redistribute
234
+ assert.strictEqual(self.receivedTickets.length, 0);
235
+ self.ticketTask.maintainDistribution();
236
+ await tick();
237
+ assert.strictEqual(peer2.receivedTickets.length, 0);
238
+ });
239
+ it("should redistribute ticket if validation callback returns true", async () => {
240
+ const self = await init("self");
241
+ const peer1 = await init("peer1");
242
+ const peer2 = await init("peer2");
243
+ self.openConnection(peer1);
244
+ self.openConnection(peer2);
245
+ await tick();
246
+ // Validation always accepts
247
+ self.ticketTask.setOnTicketReceived(async () => true);
248
+ const ticket = createTestTicket(0);
249
+ peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
250
+ peer1.ticketTask.maintainDistribution();
251
+ await tick();
252
+ // self.addTicket WAS called (callback returned true)
253
+ assert.strictEqual(self.receivedTickets.length, 1);
254
+ self.ticketTask.maintainDistribution();
255
+ await tick();
256
+ assert.strictEqual(peer2.receivedTickets.length, 1);
257
+ assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
258
+ });
220
259
  });
@@ -1 +1 @@
1
- {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAe/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAKnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CAyKlB"}
1
+ {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAe/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAKnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA0KlB"}
@@ -10,7 +10,7 @@ import { Listener } from "#@typeberry/listener";
10
10
  import { tryAsU16, tryAsU32 } from "#@typeberry/numbers";
11
11
  import { CURRENT_SUITE, CURRENT_VERSION, Result, version } from "#@typeberry/utils";
12
12
  import { DirectPort, DirectWorkerConfig } from "#@typeberry/workers-api";
13
- import { InMemWorkerConfig, LmdbWorkerConfig, ThreadPort } from "#@typeberry/workers-api-node";
13
+ import { InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort } from "#@typeberry/workers-api-node";
14
14
  import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
15
15
  import { initializeExtensions } from "./extensions.js";
16
16
  import * as metrics from "./metrics.js";
@@ -24,6 +24,7 @@ export async function main(config, withRelPath, telemetry) {
24
24
  logger.info `🫐 Typeberry ${version}. GP: ${CURRENT_VERSION} (${CURRENT_SUITE})`;
25
25
  logger.info `🎸 Starting node: ${config.nodeName}.`;
26
26
  logger.info `🖥️ PVM Backend: ${PvmBackend[config.pvmBackend]}.`;
27
+ logHostEnvironment(logger);
27
28
  const chainSpec = getChainSpec(config.node.flavor);
28
29
  const blake2b = await Blake2b.createHasher();
29
30
  const nodeName = config.nodeName;
@@ -1,3 +1,4 @@
1
+ import type { MessagePort } from "node:worker_threads";
1
2
  import { type Decode, type Encode } from "#@typeberry/codec";
2
3
  import { ChainSpec } from "#@typeberry/config";
3
4
  import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
@@ -37,6 +38,14 @@ export type TransferableConfig = {
37
38
  dbPath: string;
38
39
  workerPorts: [string, TransferablePort][];
39
40
  };
41
+ /**
42
+ * Collect the transferable objects (communication ports) embedded in a config.
43
+ *
44
+ * `MessagePort`s can only be transferred, not structurally cloned, so they have to
45
+ * be listed in the `postMessage` transfer list. Omitting them results in a
46
+ * `DataCloneError`.
47
+ */
48
+ export declare function configTransferList(config: TransferableConfig): MessagePort[];
40
49
  /**
41
50
  * In-memory (direct) worker using serialized state database.
42
51
  *
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;IA3ChD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACjC;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IASP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;IA3ChD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACjC;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IASP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
@@ -57,6 +57,16 @@ export class LmdbWorkerConfig {
57
57
  };
58
58
  }
59
59
  }
60
+ /**
61
+ * Collect the transferable objects (communication ports) embedded in a config.
62
+ *
63
+ * `MessagePort`s can only be transferred, not structurally cloned, so they have to
64
+ * be listed in the `postMessage` transfer list. Omitting them results in a
65
+ * `DataCloneError`.
66
+ */
67
+ export function configTransferList(config) {
68
+ return config.workerPorts.map(([, transferable]) => transferable.port);
69
+ }
60
70
  /**
61
71
  * In-memory (direct) worker using serialized state database.
62
72
  *
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=config.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.test.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,41 @@
1
+ import assert from "node:assert";
2
+ import { describe, it } from "node:test";
3
+ import { MessageChannel } from "node:worker_threads";
4
+ import { codec } from "#@typeberry/codec";
5
+ import { tinyChainSpec } from "#@typeberry/config";
6
+ import { Blake2b } from "#@typeberry/hash";
7
+ import { tryAsU32 } from "#@typeberry/numbers";
8
+ import { configTransferList, LmdbWorkerConfig } from "./config.js";
9
+ import { ThreadPort } from "./port.js";
10
+ const spec = tinyChainSpec;
11
+ describe("LmdbWorkerConfig transfer list", () => {
12
+ it("surfaces embedded worker ports so they can be transferred", async () => {
13
+ const blake2b = await Blake2b.createHasher();
14
+ const [portA, portB] = ThreadPort.pair(spec);
15
+ const config = LmdbWorkerConfig.new({
16
+ nodeName: "node",
17
+ chainSpec: spec,
18
+ workerParams: tryAsU32(7),
19
+ dbPath: "db",
20
+ blake2b,
21
+ ports: new Map([["authorship-network", portA]]),
22
+ });
23
+ const transferable = config.intoTransferable(codec.varU32);
24
+ const transferList = configTransferList(transferable);
25
+ // the single embedded comms port must be reported for transfer
26
+ assert.strictEqual(transferList.length, 1);
27
+ const sink = new MessageChannel();
28
+ try {
29
+ // reproduces the bug: a config carrying a port cannot be cloned without
30
+ // listing that port in the transfer list.
31
+ assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
32
+ // with the ports surfaced, posting succeeds.
33
+ assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
34
+ }
35
+ finally {
36
+ sink.port1.close();
37
+ sink.port2.close();
38
+ portB.close();
39
+ }
40
+ });
41
+ });
@@ -0,0 +1,32 @@
1
+ import type { Logger } from "#@typeberry/logger";
2
+ /**
3
+ * Log details about the host the node is running on (cpu, memory, platform)
4
+ * and the memory limits that apply to the process.
5
+ *
6
+ * Call this once from the main thread. Worker threads share the same OS process,
7
+ * so the host/cgroup/ulimit values are identical for them; use `logHeapLimit`
8
+ * to log the per-isolate V8 heap limit instead.
9
+ */
10
+ export declare function logHostEnvironment(logger: Logger): void;
11
+ /**
12
+ * Log just this isolate's V8 heap limit, labelled with the worker name.
13
+ *
14
+ * Worker threads get their own V8 isolate, so this can differ from the main
15
+ * thread. Everything else (cpu, host/cgroup/ulimit memory) is process-wide and
16
+ * already logged by `logHostEnvironment` on the main thread.
17
+ */
18
+ export declare function logHeapLimit(logger: Logger, workerName: string): void;
19
+ /**
20
+ * `resourceLimits` to pass to `new Worker(...)` so the worker isolate gets the
21
+ * same heap budget as the main thread.
22
+ *
23
+ * Worker threads run in their own V8 isolate and do not reliably inherit the
24
+ * main thread's `--max-old-space-size` (set via NODE_OPTIONS). Without this they
25
+ * fall back to V8's default (~2 GiB) and abort with SIGABRT (exit code 134) once
26
+ * they outgrow it. Call this on the main thread at spawn time so we propagate
27
+ * whatever limit the process was actually started with.
28
+ */
29
+ export declare function workerResourceLimits(): {
30
+ maxOldGenerationSizeMb: number;
31
+ };
32
+ //# sourceMappingURL=host-environment.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"host-environment.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/host-environment.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAIhD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAoBvD;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAErE;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,IAAI;IAAE,sBAAsB,EAAE,MAAM,CAAA;CAAE,CAIzE"}
@@ -0,0 +1,106 @@
1
+ import fs from "node:fs";
2
+ import os from "node:os";
3
+ import v8 from "node:v8";
4
+ const toGiB = (bytes) => `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
5
+ /**
6
+ * Log details about the host the node is running on (cpu, memory, platform)
7
+ * and the memory limits that apply to the process.
8
+ *
9
+ * Call this once from the main thread. Worker threads share the same OS process,
10
+ * so the host/cgroup/ulimit values are identical for them; use `logHeapLimit`
11
+ * to log the per-isolate V8 heap limit instead.
12
+ */
13
+ export function logHostEnvironment(logger) {
14
+ const cpus = os.cpus();
15
+ const cpuModel = cpus[0]?.model ?? "unknown";
16
+ logger.info `💻 Platform: ${os.platform()}/${os.arch()}, Node ${process.version}.`;
17
+ logger.info `⚙️ CPU: ${cpuModel} (${cpus.length} cores, ${os.availableParallelism()} available).`;
18
+ logger.info `🧠 Memory: ${toGiB(os.freemem())} free / ${toGiB(os.totalmem())} total (host).`;
19
+ logger.info `📦 V8 heap limit: ${toGiB(v8.getHeapStatistics().heap_size_limit)}.`;
20
+ // On Linux the process can be capped well below host memory by the container
21
+ // runtime (cgroup) or an address-space ulimit. os.totalmem() reflects neither,
22
+ // so surface them explicitly to make OOMs diagnosable.
23
+ const cgroupLimit = readCgroupMemoryLimit();
24
+ if (cgroupLimit !== null) {
25
+ logger.info `🐳 cgroup memory limit: ${toGiB(cgroupLimit)}.`;
26
+ }
27
+ const addressSpaceLimit = readAddressSpaceLimit();
28
+ if (addressSpaceLimit !== null) {
29
+ logger.info `🚧 Address space limit (ulimit -v): ${toGiB(addressSpaceLimit)}.`;
30
+ }
31
+ }
32
+ /**
33
+ * Log just this isolate's V8 heap limit, labelled with the worker name.
34
+ *
35
+ * Worker threads get their own V8 isolate, so this can differ from the main
36
+ * thread. Everything else (cpu, host/cgroup/ulimit memory) is process-wide and
37
+ * already logged by `logHostEnvironment` on the main thread.
38
+ */
39
+ export function logHeapLimit(logger, workerName) {
40
+ logger.info `📦 V8 heap limit (${workerName}): ${toGiB(v8.getHeapStatistics().heap_size_limit)}.`;
41
+ }
42
+ /**
43
+ * `resourceLimits` to pass to `new Worker(...)` so the worker isolate gets the
44
+ * same heap budget as the main thread.
45
+ *
46
+ * Worker threads run in their own V8 isolate and do not reliably inherit the
47
+ * main thread's `--max-old-space-size` (set via NODE_OPTIONS). Without this they
48
+ * fall back to V8's default (~2 GiB) and abort with SIGABRT (exit code 134) once
49
+ * they outgrow it. Call this on the main thread at spawn time so we propagate
50
+ * whatever limit the process was actually started with.
51
+ */
52
+ export function workerResourceLimits() {
53
+ return {
54
+ maxOldGenerationSizeMb: Math.floor(v8.getHeapStatistics().heap_size_limit / 1024 ** 2),
55
+ };
56
+ }
57
+ /** Read the cgroup (v2, then v1) memory limit in bytes, or null if unlimited / unavailable. */
58
+ function readCgroupMemoryLimit() {
59
+ if (os.platform() !== "linux") {
60
+ return null;
61
+ }
62
+ // cgroup v2 first, then the v1 fallback path.
63
+ for (const path of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
64
+ try {
65
+ const raw = fs.readFileSync(path, "utf8").trim();
66
+ if (raw === "max") {
67
+ return null; // cgroup v2 sentinel for "no limit"
68
+ }
69
+ const value = Number(raw);
70
+ // cgroup v1 reports a value close to 2^63 when unbounded; treat anything at or
71
+ // above host memory as effectively no limit.
72
+ if (!Number.isFinite(value) || value <= 0 || value >= os.totalmem()) {
73
+ return null;
74
+ }
75
+ return value;
76
+ }
77
+ catch {
78
+ // file missing or unreadable, fall through to the next candidate
79
+ }
80
+ }
81
+ return null;
82
+ }
83
+ /** Read the soft RLIMIT_AS (address space, i.e. `ulimit -v`) in bytes, or null if unlimited / unavailable. */
84
+ function readAddressSpaceLimit() {
85
+ if (os.platform() !== "linux") {
86
+ return null;
87
+ }
88
+ try {
89
+ const limits = fs.readFileSync("/proc/self/limits", "utf8");
90
+ const line = limits.split("\n").find((l) => l.startsWith("Max address space"));
91
+ if (line === undefined) {
92
+ return null;
93
+ }
94
+ // Columns are: name (multi-word), soft limit, hard limit, units.
95
+ const soft = line.slice("Max address space".length).trim().split(/\s+/)[0];
96
+ if (soft === undefined || soft === "unlimited") {
97
+ return null;
98
+ }
99
+ const value = Number(soft);
100
+ return Number.isFinite(value) && value > 0 ? value : null;
101
+ }
102
+ catch {
103
+ // /proc not mounted or unreadable
104
+ return null;
105
+ }
106
+ }
@@ -1,4 +1,5 @@
1
1
  export * from "./config.js";
2
+ export * from "./host-environment.js";
2
3
  export * from "./port.js";
3
4
  export * from "./protocol.js";
4
5
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
@@ -1,3 +1,4 @@
1
1
  export * from "./config.js";
2
+ export * from "./host-environment.js";
2
3
  export * from "./port.js";
3
4
  export * from "./protocol.js";
@@ -1 +1 @@
1
- {"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAKxE,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAChC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAkCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAkDD"}
1
+ {"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAsB,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAM5F,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAChC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAoCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAmDD"}
@@ -3,7 +3,8 @@ import { Listener } from "#@typeberry/listener";
3
3
  import { Level, Logger } from "#@typeberry/logger";
4
4
  import { assertNever } from "#@typeberry/utils";
5
5
  import { Channel } from "#@typeberry/workers-api";
6
- import { LmdbWorkerConfig } from "./config.js";
6
+ import { configTransferList, LmdbWorkerConfig } from "./config.js";
7
+ import { logHeapLimit, workerResourceLimits } from "./host-environment.js";
7
8
  import { ThreadPort } from "./port.js";
8
9
  const logger = Logger.new(import.meta.filename, "workers");
9
10
  /** Type of the control plane message. */
@@ -30,15 +31,17 @@ function isControlPlane(data) {
30
31
  export function spawnWorker(protocol, bootstrapPath, config, paramsEncoder) {
31
32
  logger.trace `Spawning ${protocol.name} child worker.`;
32
33
  const channel = new MessageChannel();
33
- const worker = new Worker(bootstrapPath);
34
+ const worker = new Worker(bootstrapPath, { resourceLimits: workerResourceLimits() });
34
35
  const msg = {
35
36
  kind: WorkerControlPlaneMsg.Config,
36
37
  parentPort: channel.port2,
37
38
  config: config.intoTransferable(paramsEncoder),
38
39
  };
39
40
  logger.trace `(${protocol.name}) <-- config`;
40
- // send the config down to the worker
41
- worker.postMessage(msg, [msg.parentPort]);
41
+ // send the config down to the worker. We need to transfer the parent
42
+ // communication port as well as any inter-worker ports carried in the config,
43
+ // otherwise structured clone fails with a `DataCloneError`.
44
+ worker.postMessage(msg, [msg.parentPort, ...configTransferList(msg.config)]);
42
45
  const workerFinished = new Promise((resolve, reject) => {
43
46
  worker.once("error", reject);
44
47
  worker.once("exit", (exitCode) => {
@@ -65,6 +68,7 @@ export async function initWorker(protocol, paramsDecoder) {
65
68
  // configure logger inside a worker thread
66
69
  Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG);
67
70
  logger.trace `Worker ${protocol.name} starting.`;
71
+ logHeapLimit(logger, protocol.name);
68
72
  return new Promise((resolve, reject) => {
69
73
  if (parentPort === null) {
70
74
  throw new Error(`Unable to start ${protocol.name} worker. Not running in a worker thread!`);