@typeberry/lib 0.9.0-c9f9e4d → 0.10.0-3e244fa

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/package.json +1 -1
  2. package/packages/core/collections/sorted-array.d.ts +1 -1
  3. package/packages/core/collections/sorted-array.d.ts.map +1 -1
  4. package/packages/core/collections/sorted-array.js +4 -2
  5. package/packages/core/collections/sorted-array.test.js +10 -0
  6. package/packages/core/collections/sorted-set.d.ts +1 -1
  7. package/packages/core/collections/sorted-set.d.ts.map +1 -1
  8. package/packages/core/collections/sorted-set.js +6 -2
  9. package/packages/core/collections/sorted-set.test.js +24 -0
  10. package/packages/core/utils/index.d.ts +1 -0
  11. package/packages/core/utils/index.d.ts.map +1 -1
  12. package/packages/core/utils/index.js +1 -0
  13. package/packages/core/utils/shutdown.d.ts +18 -0
  14. package/packages/core/utils/shutdown.d.ts.map +1 -0
  15. package/packages/core/utils/shutdown.js +3 -0
  16. package/packages/core/utils/shutdown.node.d.ts +5 -0
  17. package/packages/core/utils/shutdown.node.d.ts.map +1 -0
  18. package/packages/core/utils/shutdown.node.js +76 -0
  19. package/packages/core/utils/shutdown.test.d.ts +2 -0
  20. package/packages/core/utils/shutdown.test.d.ts.map +1 -0
  21. package/packages/core/utils/shutdown.test.js +164 -0
  22. package/packages/extensions/ipc/server.d.ts.map +1 -1
  23. package/packages/extensions/ipc/server.js +7 -0
  24. package/packages/jam/config-node/node-config.d.ts +20 -2
  25. package/packages/jam/config-node/node-config.d.ts.map +1 -1
  26. package/packages/jam/config-node/node-config.js +25 -3
  27. package/packages/jam/config-node/node-config.test.js +8 -1
  28. package/packages/jam/database/index.d.ts +1 -0
  29. package/packages/jam/database/index.d.ts.map +1 -1
  30. package/packages/jam/database/index.js +1 -0
  31. package/packages/jam/database/leaf-db-update.d.ts +1 -0
  32. package/packages/jam/database/leaf-db-update.d.ts.map +1 -1
  33. package/packages/jam/database/leaf-db-update.js +22 -5
  34. package/packages/jam/database/leaf-db-update.test.d.ts +2 -0
  35. package/packages/jam/database/leaf-db-update.test.d.ts.map +1 -0
  36. package/packages/jam/database/leaf-db-update.test.js +81 -0
  37. package/packages/jam/database/serialized-states-db.d.ts +4 -0
  38. package/packages/jam/database/serialized-states-db.d.ts.map +1 -1
  39. package/packages/jam/database/serialized-states-db.js +18 -2
  40. package/packages/jam/database/serialized-states-db.test.d.ts +2 -0
  41. package/packages/jam/database/serialized-states-db.test.d.ts.map +1 -0
  42. package/packages/jam/database/serialized-states-db.test.js +109 -0
  43. package/packages/jam/database/states.d.ts +11 -0
  44. package/packages/jam/database/states.d.ts.map +1 -1
  45. package/packages/jam/database/states.js +3 -0
  46. package/packages/jam/database/value-refs.d.ts +117 -0
  47. package/packages/jam/database/value-refs.d.ts.map +1 -0
  48. package/packages/jam/database/value-refs.js +206 -0
  49. package/packages/jam/database/value-refs.test.d.ts +2 -0
  50. package/packages/jam/database/value-refs.test.d.ts.map +1 -0
  51. package/packages/jam/database/value-refs.test.js +165 -0
  52. package/packages/jam/database-fjall/blocks.d.ts +25 -0
  53. package/packages/jam/database-fjall/blocks.d.ts.map +1 -0
  54. package/packages/jam/database-fjall/blocks.js +86 -0
  55. package/packages/jam/database-fjall/blocks.test.d.ts +2 -0
  56. package/packages/jam/database-fjall/blocks.test.d.ts.map +1 -0
  57. package/packages/jam/database-fjall/blocks.test.js +62 -0
  58. package/packages/jam/database-fjall/hybrid-states.d.ts +61 -10
  59. package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
  60. package/packages/jam/database-fjall/hybrid-states.js +131 -26
  61. package/packages/jam/database-fjall/hybrid-states.test.js +125 -3
  62. package/packages/jam/database-fjall/index.d.ts +2 -0
  63. package/packages/jam/database-fjall/index.d.ts.map +1 -1
  64. package/packages/jam/database-fjall/index.js +2 -0
  65. package/packages/jam/database-fjall/root.d.ts +29 -17
  66. package/packages/jam/database-fjall/root.d.ts.map +1 -1
  67. package/packages/jam/database-fjall/root.js +33 -13
  68. package/packages/jam/database-fjall/states.d.ts +31 -0
  69. package/packages/jam/database-fjall/states.d.ts.map +1 -0
  70. package/packages/jam/database-fjall/states.js +99 -0
  71. package/packages/jam/database-fjall/states.test.d.ts +2 -0
  72. package/packages/jam/database-fjall/states.test.d.ts.map +1 -0
  73. package/packages/jam/database-fjall/states.test.js +150 -0
  74. package/packages/jam/database-lmdb/hybrid-states.d.ts +12 -0
  75. package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
  76. package/packages/jam/database-lmdb/hybrid-states.js +45 -4
  77. package/packages/jam/database-lmdb/hybrid-states.test.js +94 -2
  78. package/packages/jam/database-lmdb/states.d.ts +1 -0
  79. package/packages/jam/database-lmdb/states.d.ts.map +1 -1
  80. package/packages/jam/database-lmdb/states.js +5 -0
  81. package/packages/jam/node/export.d.ts.map +1 -1
  82. package/packages/jam/node/export.js +18 -9
  83. package/packages/jam/node/main-fuzz.d.ts +4 -1
  84. package/packages/jam/node/main-fuzz.d.ts.map +1 -1
  85. package/packages/jam/node/main-fuzz.js +127 -55
  86. package/packages/jam/node/main-importer.d.ts +11 -2
  87. package/packages/jam/node/main-importer.d.ts.map +1 -1
  88. package/packages/jam/node/main-importer.js +27 -12
  89. package/packages/jam/node/main.d.ts.map +1 -1
  90. package/packages/jam/node/main.js +42 -15
  91. package/packages/jam/node/workers.d.ts +4 -4
  92. package/packages/jam/node/workers.d.ts.map +1 -1
  93. package/packages/workers/api/config.d.ts +2 -2
  94. package/packages/workers/api/config.d.ts.map +1 -1
  95. package/packages/workers/api/config.js +1 -1
  96. package/packages/workers/api/port.d.ts +5 -1
  97. package/packages/workers/api/port.d.ts.map +1 -1
  98. package/packages/workers/api/port.js +56 -13
  99. package/packages/workers/api/port.test.js +35 -0
  100. package/packages/workers/api-node/config.d.ts +58 -9
  101. package/packages/workers/api-node/config.d.ts.map +1 -1
  102. package/packages/workers/api-node/config.js +140 -18
  103. package/packages/workers/api-node/config.test.js +52 -3
  104. package/packages/workers/api-node/port.d.ts +3 -0
  105. package/packages/workers/api-node/port.d.ts.map +1 -1
  106. package/packages/workers/api-node/port.js +31 -2
  107. package/packages/workers/api-node/port.test.js +17 -0
  108. package/packages/workers/api-node/protocol.d.ts +3 -3
  109. package/packages/workers/api-node/protocol.d.ts.map +1 -1
  110. package/packages/workers/api-node/protocol.js +18 -12
  111. package/packages/workers/block-authorship/main.js +1 -1
  112. package/packages/workers/importer/finality.d.ts +4 -0
  113. package/packages/workers/importer/finality.d.ts.map +1 -1
  114. package/packages/workers/importer/finality.js +10 -1
  115. package/packages/workers/importer/finality.test.js +6 -0
  116. package/packages/workers/importer/importer.d.ts.map +1 -1
  117. package/packages/workers/importer/importer.js +3 -0
  118. package/packages/workers/importer/main.d.ts +1 -1
  119. package/packages/workers/importer/main.d.ts.map +1 -1
  120. package/packages/workers/importer/main.js +1 -1
  121. package/packages/workers/jam-network/main.js +1 -1
@@ -6,27 +6,31 @@ import { EventEmitter } from "eventemitter3";
6
6
  * so there is no need for any serialization - we simply pass the data.
7
7
  */
8
8
  export class DirectPort {
9
- events;
9
+ inbound;
10
+ outbound;
10
11
  threadId = 0;
11
12
  /** Create a pair of symmetrical inter-connected ports. */
12
13
  static pair() {
13
- const events = new EventEmitter();
14
- return [new DirectPort(events), new DirectPort(events)];
14
+ const left = createPortState();
15
+ const right = createPortState();
16
+ return [new DirectPort(left, right), new DirectPort(right, left)];
15
17
  }
16
- constructor(events) {
17
- this.events = events;
18
+ constructor(inbound, outbound) {
19
+ this.inbound = inbound;
20
+ this.outbound = outbound;
18
21
  }
19
22
  onClose(callback) {
20
- this.events.on("error", callback);
23
+ this.inbound.events.on("error", callback);
21
24
  }
22
25
  on(event, _codec, callback) {
23
26
  const trigger = (args) => {
24
27
  // we simply cast the args, since there is no encoding involved.
25
28
  callback(args);
26
29
  };
27
- this.events.on(event, trigger);
30
+ this.inbound.events.on(event, trigger);
31
+ this.flushPending(event, trigger);
28
32
  return () => {
29
- this.events.off(event, trigger);
33
+ this.inbound.events.off(event, trigger);
30
34
  };
31
35
  }
32
36
  once(event, _codec, callback) {
@@ -34,16 +38,55 @@ export class DirectPort {
34
38
  // we simply cast the args, since there is no encoding involved.
35
39
  callback(args);
36
40
  };
37
- this.events.once(event, trigger);
41
+ const pending = this.inbound.pendingMessages.get(event);
42
+ const pendingMessage = pending?.shift();
43
+ if (pending?.length === 0) {
44
+ this.inbound.pendingMessages.delete(event);
45
+ }
46
+ if (pendingMessage !== undefined) {
47
+ trigger(pendingMessage);
48
+ return () => { };
49
+ }
50
+ this.inbound.events.once(event, trigger);
38
51
  return () => {
39
- this.events.off(event, trigger);
52
+ this.inbound.events.off(event, trigger);
40
53
  };
41
54
  }
42
55
  postMessage(event, _codec, msg) {
43
- this.events.emit(event, msg);
56
+ if (this.outbound.events.listenerCount(event) === 0) {
57
+ this.queuePending(event, msg);
58
+ return;
59
+ }
60
+ this.outbound.events.emit(event, msg);
44
61
  }
45
62
  close() {
46
- this.events.emit("error", new Error("closing channel"));
47
- this.events.removeAllListeners();
63
+ this.closeState(this.inbound);
64
+ this.closeState(this.outbound);
48
65
  }
66
+ queuePending(event, msg) {
67
+ const pending = this.outbound.pendingMessages.get(event) ?? [];
68
+ pending.push(msg);
69
+ this.outbound.pendingMessages.set(event, pending);
70
+ }
71
+ flushPending(event, trigger) {
72
+ const pending = this.inbound.pendingMessages.get(event);
73
+ if (pending === undefined) {
74
+ return;
75
+ }
76
+ this.inbound.pendingMessages.delete(event);
77
+ for (const msg of pending) {
78
+ trigger(msg);
79
+ }
80
+ }
81
+ closeState(state) {
82
+ state.events.emit("error", new Error("closing channel"));
83
+ state.events.removeAllListeners();
84
+ state.pendingMessages.clear();
85
+ }
86
+ }
87
+ function createPortState() {
88
+ return {
89
+ events: new EventEmitter(),
90
+ pendingMessages: new Map(),
91
+ };
49
92
  }
@@ -25,4 +25,39 @@ describe("DirectPort", () => {
25
25
  data: tryAsU32(10),
26
26
  });
27
27
  });
28
+ it("should deliver messages sent before the listener is attached", () => {
29
+ const [txPort, rxPort] = DirectPort.pair();
30
+ txPort.postMessage("hello", codec.u32, {
31
+ responseId: "10",
32
+ data: tryAsU32(10),
33
+ });
34
+ let receivedMessage = null;
35
+ rxPort.on("hello", codec.u32, (msg) => {
36
+ receivedMessage = msg;
37
+ });
38
+ assert.deepStrictEqual(receivedMessage, {
39
+ responseId: "10",
40
+ data: tryAsU32(10),
41
+ });
42
+ });
43
+ it("should not let the sender consume its own pending message", () => {
44
+ const [txPort, rxPort] = DirectPort.pair();
45
+ txPort.postMessage("hello", codec.u32, {
46
+ responseId: "10",
47
+ data: tryAsU32(10),
48
+ });
49
+ let senderMessage = null;
50
+ txPort.on("hello", codec.u32, (msg) => {
51
+ senderMessage = msg;
52
+ });
53
+ assert.deepStrictEqual(senderMessage, null);
54
+ let receiverMessage = null;
55
+ rxPort.on("hello", codec.u32, (msg) => {
56
+ receiverMessage = msg;
57
+ });
58
+ assert.deepStrictEqual(receiverMessage, {
59
+ responseId: "10",
60
+ data: tryAsU32(10),
61
+ });
62
+ });
28
63
  });
@@ -2,10 +2,20 @@ import type { MessagePort } from "node:worker_threads";
2
2
  import { type Decode, type Encode } from "#@typeberry/codec";
3
3
  import { ChainSpec } from "#@typeberry/config";
4
4
  import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
5
+ import { FjallValuesSession } from "#@typeberry/database-fjall";
5
6
  import { Blake2b } from "#@typeberry/hash";
6
7
  import type { WorkerConfig } from "#@typeberry/workers-api";
7
8
  import { ThreadPort, type TransferablePort } from "./port.js";
8
- /** A worker config that's usable in node.js and uses LMDB database backend. */
9
+ export { FjallValuesSession };
10
+ /** Persistent regular-node backend. */
11
+ export type PersistentBackend = "lmdb" | "fjall";
12
+ /** Transferable worker config for persistent regular-node workers. */
13
+ export type PersistentWorkerConfig<T> = LmdbWorkerConfig<T> | FjallWorkerConfig<T>;
14
+ /**
15
+ * Worker config for node.js, backed by the LMDB database.
16
+ *
17
+ * @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
18
+ */
9
19
  export declare class LmdbWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
10
20
  readonly nodeName: string;
11
21
  readonly chainSpec: ChainSpec;
@@ -28,18 +38,51 @@ export declare class LmdbWorkerConfig<T = void> implements WorkerConfig<T, Block
28
38
  private constructor();
29
39
  openDatabase(options?: {
30
40
  readonly: boolean;
31
- }): RootDb<BlocksDb, SerializedStatesDb>;
41
+ }): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
42
+ /** Convert this config into a thread-transferable object. */
43
+ intoTransferable(paramsCodec: Encode<T>): TransferableConfig;
44
+ }
45
+ /** Worker config for node.js, backed by a shared fjall engine. */
46
+ export declare class FjallWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
47
+ readonly nodeName: string;
48
+ readonly chainSpec: ChainSpec;
49
+ readonly workerParams: T;
50
+ readonly dbPath: string;
51
+ readonly blake2b: Blake2b;
52
+ readonly ports: Map<string, ThreadPort>;
53
+ readonly ephemeral: boolean;
54
+ readonly cacheSizeBytes: number | undefined;
55
+ static new<T>({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes, }: {
56
+ nodeName: string;
57
+ chainSpec: ChainSpec;
58
+ workerParams: T;
59
+ dbPath: string;
60
+ blake2b: Blake2b;
61
+ ports?: Map<string, ThreadPort>;
62
+ ephemeral?: boolean;
63
+ cacheSizeBytes?: number;
64
+ }): FjallWorkerConfig<T>;
65
+ /** Restore node config from a transferable config object. */
66
+ static fromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<FjallWorkerConfig<T>>;
67
+ private constructor();
68
+ openDatabase(options?: {
69
+ readonly: boolean;
70
+ }): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
32
71
  /** Convert this config into a thread-transferable object. */
33
72
  intoTransferable(paramsCodec: Encode<T>): TransferableConfig;
34
73
  }
35
74
  /** Config that's safe to transfer between worker threads. */
36
75
  export type TransferableConfig = {
76
+ databaseBackend: PersistentBackend;
37
77
  nodeName: string;
38
78
  chainSpec: ChainSpec;
39
79
  workerParams: Uint8Array;
40
80
  dbPath: string;
41
81
  workerPorts: [string, TransferablePort][];
82
+ cacheSizeBytes?: number;
42
83
  };
84
+ /** Restore a persistent worker config from its transferable form. */
85
+ export declare function persistentConfigFromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<PersistentWorkerConfig<T>>;
43
86
  /**
44
87
  * Collect the transferable objects (communication ports) embedded in a config.
45
88
  *
@@ -69,17 +112,16 @@ export declare class InMemWorkerConfig<T = undefined> implements WorkerConfig<T,
69
112
  private constructor();
70
113
  openDatabase(_options?: {
71
114
  readonly: boolean;
72
- }): RootDb<BlocksDb, SerializedStatesDb>;
115
+ }): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
73
116
  }
74
117
  /** Persistent values store backing the hybrid config. */
75
118
  export type HybridBackend = "lmdb" | "fjall";
76
119
  /**
77
120
  * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
78
- * but large values persisted to disk (LMDB or fjall, selected by `backend`).
121
+ * but large values persisted to disk. The `backend` picks where the values go
122
+ * (lmdb or fjall).
79
123
  *
80
- * The fjall backend is opt-in so its performance can be compared against LMDB
81
- * before committing to it. fjall opens its keyspace asynchronously, hence the
82
- * async `new`.
124
+ * fjall opens its keyspace asynchronously, that is why `new` here is async.
83
125
  *
84
126
  * Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
85
127
  * open/close/reopen dance that genesis init performs, so `openDatabase`
@@ -99,7 +141,7 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
99
141
  readonly ephemeral: boolean;
100
142
  readonly compression: boolean;
101
143
  private readonly states;
102
- static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, backend, }: {
144
+ static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, backend, sharedFjallSession, }: {
103
145
  nodeName: string;
104
146
  chainSpec: ChainSpec;
105
147
  workerParams: T;
@@ -108,11 +150,18 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
108
150
  ephemeral?: boolean;
109
151
  compression?: boolean;
110
152
  backend?: HybridBackend;
153
+ /**
154
+ * Reuse an already-open fjall values session instead of opening a fresh
155
+ * keyspace. The fuzz target opens one per run and passes it on every reset,
156
+ * so only the in-memory blocks/leaf sets are rebuilt per vector. Ignored
157
+ * unless `backend === "fjall"`.
158
+ */
159
+ sharedFjallSession?: FjallValuesSession;
111
160
  }): Promise<HybridWorkerConfig<T>>;
112
161
  private readonly blocks;
113
162
  private constructor();
114
163
  openDatabase(_options?: {
115
164
  readonly: boolean;
116
- }): RootDb<BlocksDb, SerializedStatesDb>;
165
+ }): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
117
166
  }
118
167
  //# sourceMappingURL=config.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAQ7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAyC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAI9B,SAAS,EAAE,OAAO;IAjDpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IAaP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAavG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG;AAED,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;AAE7C;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAgCnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;aAClB,WAAW,EAAE,OAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;WAtCZ,GAAG,CAAC,CAAC,EAAE,EAClB,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,EACjB,WAAkB,EAClB,OAAgB,GACjB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,OAAO,CAAC,EAAE,aAAa,CAAC;KACzB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAUlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IAExC,OAAO;IAaP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CASzG"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAKL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAOnC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAI9D,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,uCAAuC;AACvC,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,OAAO,CAAC;AAEjD,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAEnF;;;;GAIG;AACH,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAI9B,SAAS,EAAE,OAAO;IA/CpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAgBpF,OAAO;IAaD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAahD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAU7D;AAED,kEAAkE;AAClE,qBAAa,iBAAiB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0C7E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAE9B,SAAS,EAAE,OAAO;aAClB,cAAc,EAAE,MAAM,GAAG,SAAS;IAjDpD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,EACjB,cAAc,GACf,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAiBpF,OAAO;IAYD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAuChD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAW7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,EAAE,iBAAiB,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAgBF,qEAAqE;AACrE,wBAAsB,gCAAgC,CAAC,CAAC,EACtD,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EACvB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAOpC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CAQjD;AAED,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;AAE7C;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0CnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;aAClB,WAAW,EAAE,OAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;WAhDZ,GAAG,CAAC,CAAC,EAAE,EAClB,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,EACjB,WAAkB,EAClB,OAAgB,EAChB,kBAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,OAAO,CAAC,EAAE,aAAa,CAAC;QACxB;;;;;WAKG;QACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KACzC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAYlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IAExC,OAAO;IAaD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CASjD"}
@@ -1,11 +1,18 @@
1
1
  import { Decoder, Encoder } from "#@typeberry/codec";
2
2
  import { ChainSpec } from "#@typeberry/config";
3
3
  import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
4
- import { HybridSerializedStates as FjallHybridSerializedStates } from "#@typeberry/database-fjall";
4
+ import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
5
5
  import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
6
6
  import { Blake2b } from "#@typeberry/hash";
7
7
  import { ThreadPort } from "./port.js";
8
- /** A worker config that's usable in node.js and uses LMDB database backend. */
8
+ // Re-exported so the fuzz target can open one values session per run and reuse
9
+ // it across resets (see `HybridWorkerConfig` / `mainFuzz`).
10
+ export { FjallValuesSession };
11
+ /**
12
+ * Worker config for node.js, backed by the LMDB database.
13
+ *
14
+ * @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
15
+ */
9
16
  export class LmdbWorkerConfig {
10
17
  nodeName;
11
18
  chainSpec;
@@ -19,10 +26,10 @@ export class LmdbWorkerConfig {
19
26
  }
20
27
  /** Restore node config from a transferable config object. */
21
28
  static async fromTransferable(decodeParams, config) {
22
- const blake2b = await Blake2b.createHasher();
23
- const chainSpec = ChainSpec.new(config.chainSpec);
24
- const workerParams = Decoder.decodeObject(decodeParams, config.workerParams, chainSpec);
25
- const ports = new Map(config.workerPorts.map(([name, port]) => [name, ThreadPort.fromTransferable(chainSpec, port)]));
29
+ if (config.databaseBackend !== "lmdb") {
30
+ throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
31
+ }
32
+ const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
26
33
  return LmdbWorkerConfig.new({
27
34
  nodeName: config.nodeName,
28
35
  chainSpec,
@@ -33,7 +40,7 @@ export class LmdbWorkerConfig {
33
40
  });
34
41
  }
35
42
  constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
36
- // When set, the underlying LMDB skips fsync. Only safe for throwaway
43
+ // When set, the underlying database skips fsync. Only safe for throwaway
37
44
  // databases (the fuzz target wipes on reset). Not transferred to worker
38
45
  // threads, so the durable main node path always gets the default.
39
46
  ephemeral = false) {
@@ -45,7 +52,7 @@ export class LmdbWorkerConfig {
45
52
  this.ports = ports;
46
53
  this.ephemeral = ephemeral;
47
54
  }
48
- openDatabase(options = { readonly: true }) {
55
+ async openDatabase(options = { readonly: true }) {
49
56
  const lmdb = LmdbRoot.new(this.dbPath, {
50
57
  readOnly: options.readonly,
51
58
  ephemeral: this.ephemeral,
@@ -59,14 +66,128 @@ export class LmdbWorkerConfig {
59
66
  /** Convert this config into a thread-transferable object. */
60
67
  intoTransferable(paramsCodec) {
61
68
  return {
69
+ databaseBackend: "lmdb",
70
+ nodeName: this.nodeName,
71
+ chainSpec: this.chainSpec,
72
+ workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
73
+ dbPath: this.dbPath,
74
+ workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
75
+ };
76
+ }
77
+ }
78
+ /** Worker config for node.js, backed by a shared fjall engine. */
79
+ export class FjallWorkerConfig {
80
+ nodeName;
81
+ chainSpec;
82
+ workerParams;
83
+ dbPath;
84
+ blake2b;
85
+ ports;
86
+ ephemeral;
87
+ cacheSizeBytes;
88
+ static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, }) {
89
+ return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes);
90
+ }
91
+ /** Restore node config from a transferable config object. */
92
+ static async fromTransferable(decodeParams, config) {
93
+ if (config.databaseBackend !== "fjall") {
94
+ throw new Error(`Expected fjall worker config, got ${config.databaseBackend}.`);
95
+ }
96
+ const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
97
+ return FjallWorkerConfig.new({
98
+ nodeName: config.nodeName,
99
+ chainSpec,
100
+ workerParams,
101
+ dbPath: config.dbPath,
102
+ blake2b,
103
+ ports,
104
+ cacheSizeBytes: config.cacheSizeBytes,
105
+ });
106
+ }
107
+ constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
108
+ // Kept for the fuzz/importer path. When set, persist() is skipped.
109
+ ephemeral = false, cacheSizeBytes = undefined) {
110
+ this.nodeName = nodeName;
111
+ this.chainSpec = chainSpec;
112
+ this.workerParams = workerParams;
113
+ this.dbPath = dbPath;
114
+ this.blake2b = blake2b;
115
+ this.ports = ports;
116
+ this.ephemeral = ephemeral;
117
+ this.cacheSizeBytes = cacheSizeBytes;
118
+ }
119
+ async openDatabase(options = { readonly: true }) {
120
+ const fjall = await FjallRoot.open(this.dbPath, {
121
+ readOnly: options.readonly,
122
+ ephemeral: this.ephemeral,
123
+ cacheSizeBytes: this.cacheSizeBytes,
124
+ });
125
+ let blocks = null;
126
+ let states = null;
127
+ try {
128
+ [blocks, states] = await Promise.all([
129
+ FjallBlocks.open(this.chainSpec, fjall),
130
+ FjallStates.open(this.chainSpec, this.blake2b, fjall),
131
+ ]);
132
+ }
133
+ catch (e) {
134
+ await fjall.close();
135
+ throw e;
136
+ }
137
+ return {
138
+ getBlocksDb: () => {
139
+ if (blocks === null) {
140
+ throw new Error("Fjall database is closed.");
141
+ }
142
+ return blocks;
143
+ },
144
+ getStatesDb: () => {
145
+ if (states === null) {
146
+ throw new Error("Fjall database is closed.");
147
+ }
148
+ return states;
149
+ },
150
+ close: async () => {
151
+ blocks = null;
152
+ states = null;
153
+ await fjall.close();
154
+ },
155
+ };
156
+ }
157
+ /** Convert this config into a thread-transferable object. */
158
+ intoTransferable(paramsCodec) {
159
+ return {
160
+ databaseBackend: "fjall",
62
161
  nodeName: this.nodeName,
63
162
  chainSpec: this.chainSpec,
64
163
  workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
65
164
  dbPath: this.dbPath,
66
165
  workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
166
+ cacheSizeBytes: this.cacheSizeBytes,
67
167
  };
68
168
  }
69
169
  }
170
+ async function decodeTransferableConfig(decodeParams, config) {
171
+ const blake2b = await Blake2b.createHasher();
172
+ const chainSpec = ChainSpec.new(config.chainSpec);
173
+ const workerParams = Decoder.decodeObject(decodeParams, config.workerParams, chainSpec);
174
+ const ports = new Map(config.workerPorts.map(([name, port]) => [name, ThreadPort.fromTransferable(chainSpec, port)]));
175
+ return {
176
+ blake2b,
177
+ chainSpec,
178
+ workerParams,
179
+ ports,
180
+ };
181
+ }
182
+ /** Restore a persistent worker config from its transferable form. */
183
+ export async function persistentConfigFromTransferable(decodeParams, config) {
184
+ switch (config.databaseBackend) {
185
+ case "lmdb":
186
+ return LmdbWorkerConfig.fromTransferable(decodeParams, config);
187
+ case "fjall":
188
+ return FjallWorkerConfig.fromTransferable(decodeParams, config);
189
+ }
190
+ }
70
191
  /**
71
192
  * Collect the transferable objects (communication ports) embedded in a config.
72
193
  *
@@ -100,7 +221,7 @@ export class InMemWorkerConfig {
100
221
  this.blocks = InMemoryBlocks.new();
101
222
  this.states = InMemorySerializedStates.withHasher({ chainSpec, blake2b });
102
223
  }
103
- openDatabase(_options = { readonly: true }) {
224
+ async openDatabase(_options = { readonly: true }) {
104
225
  // opening/closing db doesn't do anything, we persist the state.
105
226
  return {
106
227
  getBlocksDb: () => this.blocks,
@@ -111,11 +232,10 @@ export class InMemWorkerConfig {
111
232
  }
112
233
  /**
113
234
  * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
114
- * but large values persisted to disk (LMDB or fjall, selected by `backend`).
235
+ * but large values persisted to disk. The `backend` picks where the values go
236
+ * (lmdb or fjall).
115
237
  *
116
- * The fjall backend is opt-in so its performance can be compared against LMDB
117
- * before committing to it. fjall opens its keyspace asynchronously, hence the
118
- * async `new`.
238
+ * fjall opens its keyspace asynchronously, that is why `new` here is async.
119
239
  *
120
240
  * Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
121
241
  * open/close/reopen dance that genesis init performs, so `openDatabase`
@@ -135,11 +255,13 @@ export class HybridWorkerConfig {
135
255
  ephemeral;
136
256
  compression;
137
257
  states;
138
- static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", }) {
139
- // fjall opens its keyspace asynchronously; LMDB is synchronous. Either way
140
- // the values store is created once here and shared across reopen.
258
+ static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", sharedFjallSession, }) {
259
+ // The values store is created once here and shared across reopen. When a
260
+ // session is given (fuzz reset reuse) we wrap it instead of opening a new one.
141
261
  const states = backend === "fjall"
142
- ? await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
262
+ ? sharedFjallSession !== undefined
263
+ ? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
264
+ : await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
143
265
  : LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
144
266
  return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
145
267
  }
@@ -155,7 +277,7 @@ export class HybridWorkerConfig {
155
277
  this.states = states;
156
278
  this.blocks = InMemoryBlocks.new();
157
279
  }
158
- openDatabase(_options = { readonly: true }) {
280
+ async openDatabase(_options = { readonly: true }) {
159
281
  return {
160
282
  getBlocksDb: () => this.blocks,
161
283
  getStatesDb: () => this.states,
@@ -2,11 +2,12 @@ import assert from "node:assert";
2
2
  import * as fs from "node:fs";
3
3
  import { describe, it } from "node:test";
4
4
  import { MessageChannel } from "node:worker_threads";
5
+ import { Bytes } from "#@typeberry/bytes";
5
6
  import { codec } from "#@typeberry/codec";
6
7
  import { tinyChainSpec } from "#@typeberry/config";
7
- import { Blake2b } from "#@typeberry/hash";
8
+ import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
8
9
  import { tryAsU32 } from "#@typeberry/numbers";
9
- import { configTransferList, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
10
+ import { configTransferList, FjallWorkerConfig, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
10
11
  import { ThreadPort } from "./port.js";
11
12
  const spec = tinyChainSpec;
12
13
  describe("LmdbWorkerConfig transfer list", () => {
@@ -40,6 +41,54 @@ describe("LmdbWorkerConfig transfer list", () => {
40
41
  }
41
42
  });
42
43
  });
44
+ describe("FjallWorkerConfig transfer list", () => {
45
+ it("surfaces embedded worker ports and marks the backend", async () => {
46
+ const blake2b = await Blake2b.createHasher();
47
+ const [portA, portB] = ThreadPort.pair(spec);
48
+ const config = FjallWorkerConfig.new({
49
+ nodeName: "node",
50
+ chainSpec: spec,
51
+ workerParams: tryAsU32(7),
52
+ dbPath: "db",
53
+ blake2b,
54
+ ports: new Map([["authorship-network", portA]]),
55
+ });
56
+ try {
57
+ const transferable = config.intoTransferable(codec.varU32);
58
+ assert.strictEqual(transferable.databaseBackend, "fjall");
59
+ assert.strictEqual(configTransferList(transferable).length, 1);
60
+ }
61
+ finally {
62
+ portA.close();
63
+ portB.close();
64
+ }
65
+ });
66
+ it("opens writable and read-only handles over one shared fjall path", async () => {
67
+ const blake2b = await Blake2b.createHasher();
68
+ const dbPath = fs.mkdtempSync("typeberry-fjall-worker-");
69
+ const config = FjallWorkerConfig.new({
70
+ nodeName: "node",
71
+ chainSpec: spec,
72
+ workerParams: undefined,
73
+ dbPath,
74
+ blake2b,
75
+ });
76
+ let writer = null;
77
+ let reader = null;
78
+ try {
79
+ writer = await config.openDatabase({ readonly: false });
80
+ reader = await config.openDatabase({ readonly: true });
81
+ const best = Bytes.fill(HASH_SIZE, 9).asOpaque();
82
+ await writer.getBlocksDb().setBestHeaderHash(best);
83
+ assert.strictEqual(reader.getBlocksDb().getBestHeaderHash().toString(), best.toString());
84
+ }
85
+ finally {
86
+ await reader?.close();
87
+ await writer?.close();
88
+ fs.rmSync(dbPath, { recursive: true, force: true });
89
+ }
90
+ });
91
+ });
43
92
  describe("HybridWorkerConfig", () => {
44
93
  // Both persistent backends must construct asynchronously and hand out a
45
94
  // working db. fjall is the experimental backend we want to benchmark.
@@ -57,7 +106,7 @@ describe("HybridWorkerConfig", () => {
57
106
  ephemeral: true,
58
107
  backend,
59
108
  });
60
- const db = config.openDatabase({ readonly: false });
109
+ const db = await config.openDatabase({ readonly: false });
61
110
  const states = db.getStatesDb();
62
111
  try {
63
112
  assert.notStrictEqual(db.getBlocksDb(), undefined);
@@ -18,6 +18,7 @@ export declare class ThreadPort implements Port {
18
18
  static pair(spec: ChainSpec): [ThreadPort, ThreadPort];
19
19
  readonly threadId: number;
20
20
  private readonly events;
21
+ private readonly pendingMessages;
21
22
  static new(spec: ChainSpec, port: MessagePort): ThreadPort;
22
23
  private constructor();
23
24
  static fromTransferable(spec: ChainSpec, { port }: TransferablePort): ThreadPort;
@@ -28,5 +29,7 @@ export declare class ThreadPort implements Port {
28
29
  on<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
29
30
  once<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
30
31
  postMessage<T>(event: string, codec: Codec<T>, msg: Envelope<T>): void;
32
+ private queuePending;
33
+ private flushPending;
31
34
  }
32
35
  //# sourceMappingURL=port.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAErE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAIF,qDAAqD;AACrD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,qBAAa,UAAW,YAAW,IAAI;IAenC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAfvB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC;IAMtD,SAAgB,QAAQ,SAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW;IAI7C,OAAO;IAgBP,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,gBAAgB;IAInE,gBAAgB,IAAI,gBAAgB;IAIpC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI3C,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IASvF,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IASzF,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;CAWvE"}
1
+ {"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAErE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAIF,qDAAqD;AACrD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,qBAAa,UAAW,YAAW,IAAI;IAgBnC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAhBvB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC;IAMtD,SAAgB,QAAQ,SAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAEhE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW;IAI7C,OAAO;IAmBP,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,gBAAgB;IAInE,gBAAgB,IAAI,gBAAgB;IAIpC,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI3C,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IAUvF,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IAoBzF,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;IAYtE,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,YAAY;CAUrB"}