@typeberry/lib 0.8.4-faebc7a → 0.9.0-1c3f157

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/package.json +1 -1
  2. package/packages/configs/index.d.ts +30 -1
  3. package/packages/configs/index.d.ts.map +1 -1
  4. package/packages/configs/index.js +4 -2
  5. package/packages/configs/typeberry-dev-full.json +29 -0
  6. package/packages/core/bytes/bytes.d.ts +1 -0
  7. package/packages/core/bytes/bytes.d.ts.map +1 -1
  8. package/packages/core/bytes/bytes.js +8 -0
  9. package/packages/core/collections/sorted-array.d.ts +1 -1
  10. package/packages/core/collections/sorted-array.d.ts.map +1 -1
  11. package/packages/core/collections/sorted-array.js +4 -2
  12. package/packages/core/collections/sorted-array.test.js +10 -0
  13. package/packages/core/collections/sorted-set.d.ts +1 -1
  14. package/packages/core/collections/sorted-set.d.ts.map +1 -1
  15. package/packages/core/collections/sorted-set.js +6 -2
  16. package/packages/core/collections/sorted-set.test.js +24 -0
  17. package/packages/core/utils/debug.d.ts +4 -2
  18. package/packages/core/utils/debug.d.ts.map +1 -1
  19. package/packages/core/utils/debug.js +18 -13
  20. package/packages/core/utils/debug.test.js +12 -6
  21. package/packages/core/utils/index.d.ts +1 -0
  22. package/packages/core/utils/index.d.ts.map +1 -1
  23. package/packages/core/utils/index.js +1 -0
  24. package/packages/core/utils/shutdown.d.ts +18 -0
  25. package/packages/core/utils/shutdown.d.ts.map +1 -0
  26. package/packages/core/utils/shutdown.js +3 -0
  27. package/packages/core/utils/shutdown.node.d.ts +5 -0
  28. package/packages/core/utils/shutdown.node.d.ts.map +1 -0
  29. package/packages/core/utils/shutdown.node.js +76 -0
  30. package/packages/core/utils/shutdown.test.d.ts +2 -0
  31. package/packages/core/utils/shutdown.test.d.ts.map +1 -0
  32. package/packages/core/utils/shutdown.test.js +164 -0
  33. package/packages/extensions/ipc/server.d.ts.map +1 -1
  34. package/packages/extensions/ipc/server.js +7 -0
  35. package/packages/jam/config-node/node-config.d.ts +22 -3
  36. package/packages/jam/config-node/node-config.d.ts.map +1 -1
  37. package/packages/jam/config-node/node-config.js +33 -6
  38. package/packages/jam/config-node/node-config.test.js +11 -4
  39. package/packages/jam/database/index.d.ts +1 -0
  40. package/packages/jam/database/index.d.ts.map +1 -1
  41. package/packages/jam/database/index.js +1 -0
  42. package/packages/jam/database/leaf-db-update.d.ts +1 -0
  43. package/packages/jam/database/leaf-db-update.d.ts.map +1 -1
  44. package/packages/jam/database/leaf-db-update.js +22 -5
  45. package/packages/jam/database/leaf-db-update.test.d.ts +2 -0
  46. package/packages/jam/database/leaf-db-update.test.d.ts.map +1 -0
  47. package/packages/jam/database/leaf-db-update.test.js +81 -0
  48. package/packages/jam/database/serialized-states-db.d.ts +1 -0
  49. package/packages/jam/database/serialized-states-db.d.ts.map +1 -1
  50. package/packages/jam/database/serialized-states-db.js +4 -0
  51. package/packages/jam/database/states.d.ts +11 -0
  52. package/packages/jam/database/states.d.ts.map +1 -1
  53. package/packages/jam/database/states.js +3 -0
  54. package/packages/jam/database/value-refs.d.ts +117 -0
  55. package/packages/jam/database/value-refs.d.ts.map +1 -0
  56. package/packages/jam/database/value-refs.js +206 -0
  57. package/packages/jam/database/value-refs.test.d.ts +2 -0
  58. package/packages/jam/database/value-refs.test.d.ts.map +1 -0
  59. package/packages/jam/database/value-refs.test.js +165 -0
  60. package/packages/jam/database-fjall/blocks.d.ts +25 -0
  61. package/packages/jam/database-fjall/blocks.d.ts.map +1 -0
  62. package/packages/jam/database-fjall/blocks.js +86 -0
  63. package/packages/jam/database-fjall/blocks.test.d.ts +2 -0
  64. package/packages/jam/database-fjall/blocks.test.d.ts.map +1 -0
  65. package/packages/jam/database-fjall/blocks.test.js +62 -0
  66. package/packages/jam/database-fjall/hybrid-states.d.ts +61 -10
  67. package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
  68. package/packages/jam/database-fjall/hybrid-states.js +131 -26
  69. package/packages/jam/database-fjall/hybrid-states.test.js +31 -1
  70. package/packages/jam/database-fjall/index.d.ts +2 -0
  71. package/packages/jam/database-fjall/index.d.ts.map +1 -1
  72. package/packages/jam/database-fjall/index.js +2 -0
  73. package/packages/jam/database-fjall/root.d.ts +29 -17
  74. package/packages/jam/database-fjall/root.d.ts.map +1 -1
  75. package/packages/jam/database-fjall/root.js +33 -13
  76. package/packages/jam/database-fjall/states.d.ts +31 -0
  77. package/packages/jam/database-fjall/states.d.ts.map +1 -0
  78. package/packages/jam/database-fjall/states.js +99 -0
  79. package/packages/jam/database-fjall/states.test.d.ts +2 -0
  80. package/packages/jam/database-fjall/states.test.d.ts.map +1 -0
  81. package/packages/jam/database-fjall/states.test.js +150 -0
  82. package/packages/jam/database-lmdb/hybrid-states.d.ts +12 -0
  83. package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
  84. package/packages/jam/database-lmdb/hybrid-states.js +45 -4
  85. package/packages/jam/database-lmdb/states.d.ts +1 -0
  86. package/packages/jam/database-lmdb/states.d.ts.map +1 -1
  87. package/packages/jam/database-lmdb/states.js +5 -0
  88. package/packages/jam/jamnp-s/tasks/ticket-distribution.js +1 -1
  89. package/packages/jam/node/export.d.ts.map +1 -1
  90. package/packages/jam/node/export.js +18 -9
  91. package/packages/jam/node/main-fuzz.d.ts +4 -1
  92. package/packages/jam/node/main-fuzz.d.ts.map +1 -1
  93. package/packages/jam/node/main-fuzz.js +127 -55
  94. package/packages/jam/node/main-importer.d.ts +11 -2
  95. package/packages/jam/node/main-importer.d.ts.map +1 -1
  96. package/packages/jam/node/main-importer.js +27 -12
  97. package/packages/jam/node/main.d.ts.map +1 -1
  98. package/packages/jam/node/main.js +42 -15
  99. package/packages/jam/node/workers.d.ts +4 -4
  100. package/packages/jam/node/workers.d.ts.map +1 -1
  101. package/packages/jam/safrole/bandersnatch-vrf.d.ts +22 -2
  102. package/packages/jam/safrole/bandersnatch-vrf.d.ts.map +1 -1
  103. package/packages/jam/safrole/bandersnatch-vrf.js +54 -20
  104. package/packages/jam/safrole/bandersnatch-vrf.test.js +3 -2
  105. package/packages/jam/safrole/bandersnatch-wasm.d.ts +10 -0
  106. package/packages/jam/safrole/bandersnatch-wasm.d.ts.map +1 -1
  107. package/packages/jam/safrole/bandersnatch-wasm.js +12 -0
  108. package/packages/jam/ticket-pool/ticket-validator.d.ts +5 -4
  109. package/packages/jam/ticket-pool/ticket-validator.d.ts.map +1 -1
  110. package/packages/jam/ticket-pool/ticket-validator.js +8 -3
  111. package/packages/jam/ticket-pool/ticket-validator.test.js +5 -4
  112. package/packages/jam/ticket-pool/verified-ticket-pool.d.ts +2 -0
  113. package/packages/jam/ticket-pool/verified-ticket-pool.d.ts.map +1 -1
  114. package/packages/jam/ticket-pool/verified-ticket-pool.js +4 -0
  115. package/packages/jam/ticket-pool/verified-ticket-pool.test.js +5 -5
  116. package/packages/workers/api/config.d.ts +2 -2
  117. package/packages/workers/api/config.d.ts.map +1 -1
  118. package/packages/workers/api/config.js +1 -1
  119. package/packages/workers/api/port.d.ts +5 -1
  120. package/packages/workers/api/port.d.ts.map +1 -1
  121. package/packages/workers/api/port.js +56 -13
  122. package/packages/workers/api/port.test.js +35 -0
  123. package/packages/workers/api-node/config.d.ts +58 -9
  124. package/packages/workers/api-node/config.d.ts.map +1 -1
  125. package/packages/workers/api-node/config.js +140 -18
  126. package/packages/workers/api-node/config.test.js +52 -3
  127. package/packages/workers/api-node/port.d.ts +3 -0
  128. package/packages/workers/api-node/port.d.ts.map +1 -1
  129. package/packages/workers/api-node/port.js +31 -2
  130. package/packages/workers/api-node/port.test.js +17 -0
  131. package/packages/workers/api-node/protocol.d.ts +3 -3
  132. package/packages/workers/api-node/protocol.d.ts.map +1 -1
  133. package/packages/workers/api-node/protocol.js +18 -12
  134. package/packages/workers/block-authorship/{generator.d.ts → block-generator.d.ts} +5 -5
  135. package/packages/workers/block-authorship/block-generator.d.ts.map +1 -0
  136. package/packages/workers/block-authorship/{generator.js → block-generator.js} +3 -3
  137. package/packages/workers/block-authorship/block-generator.test.d.ts +2 -0
  138. package/packages/workers/block-authorship/block-generator.test.d.ts.map +1 -0
  139. package/packages/workers/block-authorship/{generator.test.js → block-generator.test.js} +8 -8
  140. package/packages/workers/block-authorship/epoch-authoring-slots.d.ts +35 -0
  141. package/packages/workers/block-authorship/epoch-authoring-slots.d.ts.map +1 -0
  142. package/packages/workers/block-authorship/epoch-authoring-slots.js +86 -0
  143. package/packages/workers/block-authorship/epoch-tracker.d.ts +29 -0
  144. package/packages/workers/block-authorship/epoch-tracker.d.ts.map +1 -0
  145. package/packages/workers/block-authorship/epoch-tracker.js +80 -0
  146. package/packages/workers/block-authorship/index.d.ts.map +1 -1
  147. package/packages/workers/block-authorship/index.js +1 -1
  148. package/packages/workers/block-authorship/main.d.ts +3 -0
  149. package/packages/workers/block-authorship/main.d.ts.map +1 -1
  150. package/packages/workers/block-authorship/main.js +194 -262
  151. package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts +2 -0
  152. package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts.map +1 -0
  153. package/packages/workers/block-authorship/ticket-generator/bootstrap-main.js +23 -0
  154. package/packages/workers/block-authorship/ticket-generator/index.d.ts +16 -0
  155. package/packages/workers/block-authorship/ticket-generator/index.d.ts.map +1 -0
  156. package/packages/workers/block-authorship/ticket-generator/index.js +62 -0
  157. package/packages/workers/block-authorship/ticket-generator/protocol.d.ts +50 -0
  158. package/packages/workers/block-authorship/ticket-generator/protocol.d.ts.map +1 -0
  159. package/packages/workers/block-authorship/ticket-generator/protocol.js +54 -0
  160. package/packages/workers/block-authorship/{ticket-generator.d.ts → ticket-generator/ticket-generator.d.ts} +4 -0
  161. package/packages/workers/block-authorship/ticket-generator/ticket-generator.d.ts.map +1 -0
  162. package/packages/workers/block-authorship/{ticket-generator.js → ticket-generator/ticket-generator.js} +19 -9
  163. package/packages/workers/block-authorship/ticket-generator/ticket-generator.test.d.ts.map +1 -0
  164. package/packages/workers/block-authorship/{ticket-generator.test.js → ticket-generator/ticket-generator.test.js} +13 -9
  165. package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts +36 -0
  166. package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts.map +1 -0
  167. package/packages/workers/block-authorship/ticket-generator/worker-pool.js +111 -0
  168. package/packages/workers/block-authorship/ticket-validator.d.ts +7 -8
  169. package/packages/workers/block-authorship/ticket-validator.d.ts.map +1 -1
  170. package/packages/workers/block-authorship/ticket-validator.js +26 -23
  171. package/packages/workers/comms-authorship-network/protocol.d.ts +4 -4
  172. package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
  173. package/packages/workers/comms-authorship-network/protocol.js +4 -5
  174. package/packages/workers/comms-authorship-network/tickets-message.d.ts +0 -14
  175. package/packages/workers/comms-authorship-network/tickets-message.d.ts.map +1 -1
  176. package/packages/workers/comms-authorship-network/tickets-message.js +0 -17
  177. package/packages/workers/importer/finality.d.ts +2 -0
  178. package/packages/workers/importer/finality.d.ts.map +1 -1
  179. package/packages/workers/importer/finality.js +8 -1
  180. package/packages/workers/importer/importer.d.ts +2 -2
  181. package/packages/workers/importer/importer.d.ts.map +1 -1
  182. package/packages/workers/importer/importer.js +6 -5
  183. package/packages/workers/importer/main.d.ts +1 -1
  184. package/packages/workers/importer/main.d.ts.map +1 -1
  185. package/packages/workers/importer/main.js +1 -1
  186. package/packages/workers/importer/stats.d.ts +1 -3
  187. package/packages/workers/importer/stats.d.ts.map +1 -1
  188. package/packages/workers/importer/stats.js +12 -12
  189. package/packages/workers/jam-network/main.d.ts.map +1 -1
  190. package/packages/workers/jam-network/main.js +9 -4
  191. package/packages/workers/block-authorship/generator.d.ts.map +0 -1
  192. package/packages/workers/block-authorship/generator.test.d.ts +0 -2
  193. package/packages/workers/block-authorship/generator.test.d.ts.map +0 -1
  194. package/packages/workers/block-authorship/ticket-generator.d.ts.map +0 -1
  195. package/packages/workers/block-authorship/ticket-generator.test.d.ts.map +0 -1
  196. /package/packages/configs/{typeberry-dev.json → typeberry-dev-tiny.json} +0 -0
  197. /package/packages/workers/block-authorship/{ticket-generator.test.d.ts → ticket-generator/ticket-generator.test.d.ts} +0 -0
@@ -0,0 +1,150 @@
1
+ import assert from "node:assert";
2
+ import * as fs from "node:fs";
3
+ import { afterEach, before, beforeEach, describe, it } from "node:test";
4
+ import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
5
+ import { Bytes, BytesBlob } from "#@typeberry/bytes";
6
+ import { tinyChainSpec } from "#@typeberry/config";
7
+ import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
8
+ import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
9
+ import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
10
+ import { testState } from "#@typeberry/state/test.utils.js";
11
+ import { StateEntries } from "#@typeberry/state-merkleization";
12
+ import { deepEqual, OK, Result } from "#@typeberry/utils";
13
+ import { FjallRoot } from "./root.js";
14
+ import { FjallStates } from "./states.js";
15
+ let blake2b;
16
+ before(async () => {
17
+ blake2b = await Blake2b.createHasher();
18
+ });
19
+ describe("Fjall states database", () => {
20
+ const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
21
+ const spec = tinyChainSpec;
22
+ let tmpDir = "";
23
+ beforeEach(() => {
24
+ tmpDir = fs.mkdtempSync("typeberry-fjall-states-");
25
+ });
26
+ afterEach(() => {
27
+ fs.rmSync(tmpDir, { recursive: true, force: true });
28
+ });
29
+ it("imports and reads an empty state", async () => {
30
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
31
+ const states = await FjallStates.open(spec, blake2b, root);
32
+ try {
33
+ const emptyState = InMemoryState.empty(spec);
34
+ const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
35
+ const emptyRoot = serialized.getRootHash(blake2b);
36
+ deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
37
+ const newState = states.getState(headerHash);
38
+ assert.ok(newState !== null);
39
+ assert.strictEqual(`${await states.getStateRoot(newState)}`, `${emptyRoot}`);
40
+ deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
41
+ }
42
+ finally {
43
+ await states.close();
44
+ await root.close();
45
+ }
46
+ });
47
+ it("reads large values back from disk", async () => {
48
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
49
+ const states = await FjallStates.open(spec, blake2b, root);
50
+ try {
51
+ const big1 = BytesBlob.blobFromString("x".repeat(100));
52
+ const big2 = BytesBlob.blobFromString("y".repeat(100));
53
+ const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
54
+ const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
55
+ const entries = StateEntries.fromEntriesUnsafe([
56
+ [key1, big1],
57
+ [key2, big2],
58
+ ]);
59
+ deepEqual(await states.insertInitialState(headerHash, entries), Result.ok(OK));
60
+ const state = states.getState(headerHash);
61
+ assert.ok(state !== null);
62
+ assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
63
+ assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
64
+ }
65
+ finally {
66
+ await states.close();
67
+ await root.close();
68
+ }
69
+ });
70
+ it("updates state", async () => {
71
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
72
+ const states = await FjallStates.open(spec, blake2b, root);
73
+ try {
74
+ const state = InMemoryState.empty(spec);
75
+ await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
76
+ const newState = states.getState(headerHash);
77
+ assert.ok(newState !== null);
78
+ const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
79
+ const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
80
+ const stateUpdate = {
81
+ timeslot: tryAsTimeSlot(15),
82
+ privilegedServices: PrivilegedServices.create({
83
+ manager: tryAsServiceId(1),
84
+ assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
85
+ delegator: tryAsServiceId(3),
86
+ registrar: tryAsServiceId(4),
87
+ autoAccumulateServices: new Map(),
88
+ }),
89
+ updated: new Map([
90
+ [
91
+ tryAsServiceId(1),
92
+ UpdateService.create({
93
+ serviceInfo: ServiceAccountInfo.create({
94
+ codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
95
+ balance: tryAsU64(1_000_000),
96
+ accumulateMinGas: tryAsServiceGas(10_000),
97
+ onTransferMinGas: tryAsServiceGas(5_000),
98
+ storageUtilisationBytes: tryAsU64(1_000),
99
+ gratisStorage: tryAsU64(0),
100
+ storageUtilisationCount: tryAsU32(1),
101
+ created: tryAsTimeSlot(0),
102
+ lastAccumulation: tryAsTimeSlot(0),
103
+ parentService: tryAsServiceId(0),
104
+ }),
105
+ lookupHistory,
106
+ }),
107
+ ],
108
+ ]),
109
+ };
110
+ deepEqual(state.applyUpdate(stateUpdate), Result.ok(OK));
111
+ deepEqual(await states.updateAndSetState(headerHash2, newState, stateUpdate), Result.ok(OK));
112
+ const updatedState = states.getState(headerHash2);
113
+ assert.ok(updatedState !== null);
114
+ deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
115
+ [
116
+ tryAsServiceId(1),
117
+ {
118
+ storageKeys: [],
119
+ preimages: [],
120
+ lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
121
+ },
122
+ ],
123
+ ])), state);
124
+ assert.strictEqual(`${await states.getStateRoot(updatedState)}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
125
+ }
126
+ finally {
127
+ await states.close();
128
+ await root.close();
129
+ }
130
+ });
131
+ it("imports a more complex state", async () => {
132
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
133
+ const states = await FjallStates.open(spec, blake2b, root);
134
+ try {
135
+ const initialState = testState();
136
+ const initialService = initialState.services.get(tryAsServiceId(0));
137
+ assert.ok(initialService !== undefined);
138
+ const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
139
+ deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
140
+ const newState = states.getState(headerHash);
141
+ assert.ok(newState !== null);
142
+ assert.strictEqual(`${await states.getStateRoot(newState)}`, `${serialized.getRootHash(blake2b)}`);
143
+ deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
144
+ }
145
+ finally {
146
+ await states.close();
147
+ await root.close();
148
+ }
149
+ });
150
+ });
@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
12
12
  * Reads go straight to lmdb, which keeps its own page cache.
13
13
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
14
14
  * keep the heap usage bounded.
15
+ *
16
+ * Values that no longer belong to any surviving state are removed from lmdb,
17
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
18
+ * finality signal. Counts are not persisted: this db cannot resume from disk
19
+ * anyway (the leaf sets live in memory), so values left over by a previous
20
+ * run are never collected.
15
21
  */
16
22
  export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
17
23
  private readonly spec;
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
20
26
  private readonly inMemStates;
21
27
  private readonly lmdbValues;
22
28
  private readonly valuesDb;
29
+ private readonly refsStore;
30
+ private readonly refs;
31
+ private pendingCleanup;
23
32
  static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
24
33
  spec: ChainSpec;
25
34
  blake2b: Blake2b;
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
33
42
  updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
34
43
  getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
35
44
  getState(header: HeaderHash): SerializedState<LeafDb> | null;
45
+ commitFinalized(headers: HeaderHash[]): void;
36
46
  markUnused(header: HeaderHash): void;
47
+ /** Apply a refcounting update and remove values that lost their last reference. */
48
+ private applyRefs;
37
49
  diskSizeInBytes(): number | null;
38
50
  close(): Promise<void>;
39
51
  /** Write new large values to LMDB in one transaction. */
@@ -1 +1 @@
1
- {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
1
+ {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
@@ -1,9 +1,11 @@
1
1
  import { HashDictionary, SortedSet } from "#@typeberry/collections";
2
- import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
2
+ import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
3
+ import { Logger } from "#@typeberry/logger";
3
4
  import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
4
5
  import { leafComparator } from "#@typeberry/trie";
5
6
  import { OK, Result } from "#@typeberry/utils";
6
7
  import { LmdbRoot } from "./root.js";
8
+ const logger = Logger.new(import.meta.filename, "db");
7
9
  /**
8
10
  * Hybrid serialized-states db.
9
11
  *
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
11
13
  * Reads go straight to lmdb, which keeps its own page cache.
12
14
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
13
15
  * keep the heap usage bounded.
16
+ *
17
+ * Values that no longer belong to any surviving state are removed from lmdb,
18
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
19
+ * finality signal. Counts are not persisted: this db cannot resume from disk
20
+ * anyway (the leaf sets live in memory), so values left over by a previous
21
+ * run are never collected.
14
22
  */
15
23
  export class HybridSerializedStates {
16
24
  spec;
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
20
28
  lmdbValues;
21
29
  // A single shared values accessor reused by every `LeafDb` we hand out.
22
30
  valuesDb;
31
+ refsStore = new InMemoryValueRefsStore();
32
+ refs = new ValueRefs(this.refsStore);
33
+ // Queue of not-yet-committed value removals, awaited on close.
34
+ pendingCleanup = Promise.resolve();
23
35
  static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
24
36
  const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
25
37
  return new HybridSerializedStates(spec, blake2b, root);
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
38
50
  return res;
39
51
  }
40
52
  this.inMemStates.set(headerHash, leafs);
53
+ this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
41
54
  return Result.ok(OK);
42
55
  }
43
56
  async updateAndSetState(header, state, update) {
44
57
  const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
45
58
  // Clone the leaf set before mutating: the previous state keeps using its own.
46
59
  const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
47
- const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
60
+ const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
48
61
  const res = await this.writeValues(values);
49
62
  if (res.isError) {
50
63
  // Leave the caller's state untouched: its new leaves would reference
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
55
68
  // values are durably written.
56
69
  state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
57
70
  this.inMemStates.set(header, leafs);
71
+ this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
58
72
  return Result.ok(OK);
59
73
  }
60
74
  async getStateRoot(state) {
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
68
82
  const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
69
83
  return SerializedState.new(this.spec, this.blake2b, leafDb);
70
84
  }
85
+ commitFinalized(headers) {
86
+ this.applyRefs(this.refs.commitFinalized(headers));
87
+ }
71
88
  markUnused(header) {
72
- // We only remove the state from memory - values are not pruned at all,
73
- // but since they are stored on disk we should be safe.
89
+ // Release the speculative references first (a no-op for finalized states,
90
+ // whose deltas were already consumed by `commitFinalized`).
91
+ this.applyRefs(this.refs.releaseUnfinalized(header));
74
92
  this.inMemStates.delete(header);
75
93
  }
94
+ /** Apply a refcounting update and remove values that lost their last reference. */
95
+ applyRefs(update) {
96
+ this.refsStore.apply(update);
97
+ if (update.removeValues.length === 0) {
98
+ return;
99
+ }
100
+ // Queued, not awaited: a failed removal only leaks a value.
101
+ this.pendingCleanup = this.pendingCleanup
102
+ .then(() => this.lmdbValues.transaction(() => {
103
+ for (const v of update.removeValues) {
104
+ this.lmdbValues.remove(v.raw);
105
+ }
106
+ }))
107
+ .catch((e) => {
108
+ logger.error `Failed to remove unreferenced values: ${e}`;
109
+ });
110
+ }
76
111
  diskSizeInBytes() {
77
112
  return this.root.sizeInBytes();
78
113
  }
79
114
  async close() {
115
+ await this.pendingCleanup;
80
116
  await this.lmdbValues.close();
81
117
  await this.root.close();
82
118
  }
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
86
122
  return Result.ok(OK);
87
123
  }
88
124
  try {
125
+ // Flush queued removals first: a pending cleanup might be about to delete
126
+ // a content hash we are re-inserting now. Writing behind it keeps the
127
+ // re-inserted value on disk (removals are only queued at refcount 0, so
128
+ // none can be queued for a value still referenced).
129
+ await this.pendingCleanup;
89
130
  await this.lmdbValues.transaction(() => {
90
131
  for (const [hash, val] of values) {
91
132
  this.lmdbValues.put(hash.raw, val.raw);
@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
68
68
  updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
69
69
  getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
70
70
  getState(root: HeaderHash): SerializedState<LeafDb> | null;
71
+ commitFinalized(_headers: HeaderHash[]): void;
71
72
  markUnused(header: HeaderHash): void;
72
73
  diskSizeInBytes(): number | null;
73
74
  close(): Promise<void>;
@@ -1 +1 @@
1
- {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
1
+ {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
@@ -128,6 +128,11 @@ export class LmdbStates {
128
128
  }
129
129
  return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
130
130
  }
131
+ commitFinalized(_headers) {
132
+ // Values are never pruned here. This db survives restarts, so refcounting
133
+ // would need counts persisted (and crash-consistent) alongside the values.
134
+ // See the TODO above - not implemented until actually needed.
135
+ }
131
136
  markUnused(header) {
132
137
  this.states.removeSync(header.raw);
133
138
  }
@@ -108,7 +108,7 @@ export class TicketDistributionTask {
108
108
  // Wrap with Promise.resolve().then() so a synchronous throw inside the validator
109
109
  // funnels into the same .catch() as an async rejection.
110
110
  Promise.resolve()
111
- .then(() => validator.validate(epochIndex, ticket))
111
+ .then(() => validator.validate(epochIndex, [ticket]))
112
112
  .then((result) => {
113
113
  if (result.isOk) {
114
114
  this.addTicket(epochIndex, ticket);
@@ -1 +1 @@
1
- {"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBA2F9G"}
1
+ {"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}
@@ -3,9 +3,10 @@ import path from "node:path";
3
3
  import { Block as BlockCodec } from "#@typeberry/block";
4
4
  import { Bytes } from "#@typeberry/bytes";
5
5
  import { Encoder } from "#@typeberry/codec";
6
+ import { RegularStateBackend } from "#@typeberry/config-node";
6
7
  import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
7
8
  import { Logger } from "#@typeberry/logger";
8
- import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
9
+ import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
9
10
  import { getChainSpec, getDatabasePath } from "./common.js";
10
11
  export async function exportBlocks(jamNodeConfig, output, withRelPath) {
11
12
  const logger = Logger.new(import.meta.filename, "export");
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
29
30
  const blake2b = await Blake2b.createHasher();
30
31
  const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
31
32
  const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
32
- const config = LmdbWorkerConfig.new({
33
- nodeName: jamNodeConfig.nodeName,
34
- chainSpec,
35
- blake2b,
36
- dbPath,
37
- workerParams: undefined,
38
- });
39
- const rootDb = config.openDatabase();
33
+ const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
34
+ ? FjallWorkerConfig.new({
35
+ nodeName: jamNodeConfig.nodeName,
36
+ chainSpec,
37
+ blake2b,
38
+ dbPath,
39
+ workerParams: undefined,
40
+ })
41
+ : LmdbWorkerConfig.new({
42
+ nodeName: jamNodeConfig.nodeName,
43
+ chainSpec,
44
+ blake2b,
45
+ dbPath,
46
+ workerParams: undefined,
47
+ });
48
+ const rootDb = await config.openDatabase();
40
49
  const blocks = rootDb.getBlocksDb();
41
50
  logger.info `📖 Gathering blocks...`;
42
51
  const hashes = [];
@@ -1,5 +1,6 @@
1
1
  import { type FuzzVersion } from "#@typeberry/ext-ipc";
2
2
  import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
3
+ import { type Closer } from "#@typeberry/utils";
3
4
  import type { JamConfig } from "./jam-config.js";
4
5
  export type FuzzConfig = {
5
6
  version: FuzzVersion;
@@ -24,5 +25,7 @@ export declare function getFuzzDetails(): {
24
25
  nodeVersion: fuzzV1.Version;
25
26
  gpVersion: fuzzV1.Version;
26
27
  };
27
- export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<() => void>;
28
+ export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<{
29
+ close: Closer;
30
+ }>;
28
31
  //# sourceMappingURL=main-fuzz.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAWF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBAqHxF"}
1
+ {"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}
@@ -8,7 +8,7 @@ import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
8
8
  import { HASH_SIZE } from "#@typeberry/hash";
9
9
  import { Logger } from "#@typeberry/logger";
10
10
  import { CURRENT_VERSION, Result, version } from "#@typeberry/utils";
11
- import { logHostEnvironment } from "#@typeberry/workers-api-node";
11
+ import { FjallValuesSession, logHostEnvironment } from "#@typeberry/workers-api-node";
12
12
  import { getChainSpec } from "./common.js";
13
13
  import { mainImporter } from "./main-importer.js";
14
14
  const logger = Logger.new(import.meta.filename, "fuzztarget");
@@ -17,6 +17,14 @@ const FUZZ_DB_SUBDIR = "typeberry-fuzz-db";
17
17
  const FUZZ_DB_FJALL = "fjall-hybrid";
18
18
  const FUZZ_DB_LMDB = "lmdb-hybrid";
19
19
  const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, FUZZ_DB_LMDB];
20
+ /** Subdirectory (under the fuzzer's db dir) holding the reused fjall values keyspace. */
21
+ const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
22
+ /**
23
+ * Size of the fjall block-cache for the fuzz session. Values pile up across
24
+ * resets (for fjall we do not wipe between them), so this cache is what keeps
25
+ * the resident memory bounded.
26
+ */
27
+ const FUZZ_FJALL_CACHE_BYTES = 128 * 1024 * 1024;
20
28
  /**
21
29
  * Resolve the directory the fuzzer should use for its on-disk database, or
22
30
  * `undefined` for an in-memory database. The dedicated `FUZZ_DB_SUBDIR` is
@@ -63,6 +71,14 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
63
71
  logger.info `🗄️ Fuzz persistent backend: ${hybridStateBackend}.`;
64
72
  }
65
73
  let runningNode = null;
74
+ // The fjall values keyspace is opened once per fuzz session and reused on
75
+ // every reset, because opening it is the slow part. Only the in-memory blocks
76
+ // and leaf sets are rebuilt for each vector. fjall-hybrid only.
77
+ let fjallSession = null;
78
+ // Set when close() starts. Guards resetState so a fuzz command arriving
79
+ // mid-shutdown can't build a fresh node that close() then orphans.
80
+ let isClosing = false;
81
+ let activeReset = null;
66
82
  const chainSpec = getChainSpec(config.node.flavor);
67
83
  const closeFuzzTarget = startFuzzTarget(fuzzConfig.version, fuzzConfig.socket, {
68
84
  ...getFuzzDetails(),
@@ -86,66 +102,122 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
86
102
  }
87
103
  return runningNode.getStateEntries(hash);
88
104
  },
89
- resetState: async (header, state, ancestry) => {
105
+ resetState: (header, state, ancestry) => {
106
+ const reset = (async () => {
107
+ if (isClosing) {
108
+ return Bytes.zero(HASH_SIZE).asOpaque();
109
+ }
110
+ if (runningNode !== null) {
111
+ const finish = runningNode.close();
112
+ runningNode = null;
113
+ await finish;
114
+ }
115
+ const buildNode = (databaseBasePath) => {
116
+ const isPersistent = databaseBasePath !== undefined;
117
+ return mainImporter({
118
+ ...config,
119
+ node: {
120
+ ...config.node,
121
+ databaseBasePath,
122
+ chainSpec: {
123
+ ...config.node.chainSpec,
124
+ genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
125
+ genesisState: new Map(state),
126
+ },
127
+ },
128
+ ancestry,
129
+ network: null,
130
+ }, withRelPath, {
131
+ initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
132
+ // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
133
+ // like the in-memory backend; only the large values live on disk.
134
+ dummyFinalityDepth: 20,
135
+ pruneBlocks: true,
136
+ // The on-disk fuzz db is throwaway (we wipe it), so open it ephemeral and
137
+ // skip the fsync, we do not need durability here. On full spec ephemeral
138
+ // also turns on compression further down, so the big values do not grow the
139
+ // db too much. Tiny stays uncompressed, its db is small and speed matters more.
140
+ ephemeral: isPersistent,
141
+ stateBackend: isPersistent ? hybridStateBackend : "lmdb",
142
+ // Reuse the session keyspace (fjall-hybrid only, other backends
143
+ // ignore it). Nothing to pass for the in-memory fallback.
144
+ sharedFjallSession: isPersistent ? (fjallSession ?? undefined) : undefined,
145
+ });
146
+ };
147
+ if (fuzzDbBase !== undefined) {
148
+ try {
149
+ if (hybridStateBackend === FUZZ_DB_FJALL) {
150
+ // fjall-hybrid: open the values keyspace once and reuse it on every
151
+ // reset. The values partition is content-addressed and immutable, so
152
+ // it is fine that values pile up across resets, the unreferenced ones
153
+ // just sit there. `initializeDatabase` decides whether the db is
154
+ // already initialized from the in-memory blocks, which we rebuild on
155
+ // every reset, not from the values store, so reusing it does not
156
+ // resume the previous run.
157
+ if (fjallSession === null) {
158
+ // Start from a clean slate once, then keep the keyspace open.
159
+ await wipeFuzzDb(fuzzDbBase);
160
+ fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
161
+ ephemeral: true,
162
+ cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
163
+ });
164
+ logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
165
+ }
166
+ }
167
+ else {
168
+ // lmdb-hybrid: keep the old behaviour, wipe and reopen on every
169
+ // reset. A fresh db each reset makes `initializeDatabase` set up
170
+ // genesis again instead of resuming the previous run.
171
+ await wipeFuzzDb(fuzzDbBase);
172
+ }
173
+ runningNode = await buildNode(fuzzDbBase);
174
+ return await runningNode.getBestStateRootHash();
175
+ }
176
+ catch (e) {
177
+ // A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
178
+ logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
179
+ runningNode = null;
180
+ }
181
+ }
182
+ runningNode = await buildNode(undefined);
183
+ return await runningNode.getBestStateRootHash();
184
+ })();
185
+ activeReset = reset;
186
+ const clearActiveReset = () => {
187
+ if (activeReset === reset) {
188
+ activeReset = null;
189
+ }
190
+ };
191
+ reset.then(clearActiveReset, clearActiveReset);
192
+ return reset;
193
+ },
194
+ });
195
+ return {
196
+ close: async () => {
197
+ isClosing = true;
198
+ // Stop accepting connections + unlink the socket.
199
+ closeFuzzTarget();
200
+ // Drain the active session (flush + close DB). Swallow errors so a
201
+ // failing close still lets the process exit 0; the db is wiped next.
202
+ // The node references the shared fjall session, so it must close first.
203
+ if (activeReset !== null) {
204
+ await activeReset.catch((e) => logger.error `Error waiting for fuzz reset: ${e}`);
205
+ }
90
206
  if (runningNode !== null) {
91
- const finish = runningNode.close();
207
+ const node = runningNode;
92
208
  runningNode = null;
93
- await finish;
209
+ await node.close().catch((e) => logger.error `Error closing fuzz node: ${e}`);
210
+ }
211
+ // Release the reused fjall values keyspace before wiping its files.
212
+ if (fjallSession !== null) {
213
+ const session = fjallSession;
214
+ fjallSession = null;
215
+ await session.close().catch((e) => logger.error `Error closing fjall session: ${e}`);
94
216
  }
95
- const buildNode = (databaseBasePath) => {
96
- const isPersistent = databaseBasePath !== undefined;
97
- return mainImporter({
98
- ...config,
99
- node: {
100
- ...config.node,
101
- databaseBasePath,
102
- chainSpec: {
103
- ...config.node.chainSpec,
104
- genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
105
- genesisState: new Map(state),
106
- },
107
- },
108
- ancestry,
109
- network: null,
110
- }, withRelPath, {
111
- initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
112
- // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
113
- // like the in-memory backend; only the large values live on disk.
114
- dummyFinalityDepth: 20,
115
- pruneBlocks: true,
116
- // Long full-spec sessions accumulate a large, never-pruned values db.
117
- // Syncing lets the OS reclaim dirty mmap pages, and compression (full
118
- // spec only, where values are big) bounds its on-disk/page-cache size.
119
- // Tiny stays uncompressed since its db is small and speed matters more.
120
- ephemeral: isPersistent,
121
- stateBackend: isPersistent ? hybridStateBackend : "lmdb",
122
- });
123
- };
124
217
  if (fuzzDbBase !== undefined) {
125
- // Each reset starts a fresh session from the genesis the fuzzer just sent,
126
- // so the on-disk db must be empty: otherwise initializeDatabase sees an
127
- // already-initialized db and silently resumes the previous run's state.
128
- await wipeFuzzDb(fuzzDbBase);
129
- try {
130
- runningNode = await buildNode(fuzzDbBase);
131
- return await runningNode.getBestStateRootHash();
132
- }
133
- catch (e) {
134
- // A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
135
- logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
136
- runningNode = null;
137
- }
218
+ await wipeFuzzDb(fuzzDbBase).catch(() => { });
138
219
  }
139
- runningNode = await buildNode(undefined);
140
- return await runningNode.getBestStateRootHash();
141
220
  },
142
- });
143
- return () => {
144
- closeFuzzTarget();
145
- if (fuzzDbBase !== undefined) {
146
- // best-effort cleanup on shutdown; ignore failures (dir may already be gone).
147
- wipeFuzzDb(fuzzDbBase).catch(() => { });
148
- }
149
221
  };
150
222
  }
151
223
  function isValidStateBackend(val) {