@typeberry/lib 0.10.0-9e3af22 → 0.11.0-411c828

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/package.json +1 -3
  2. package/packages/jam/config-node/node-config.d.ts +1 -11
  3. package/packages/jam/config-node/node-config.d.ts.map +1 -1
  4. package/packages/jam/config-node/node-config.js +3 -25
  5. package/packages/jam/config-node/node-config.test.js +3 -7
  6. package/packages/jam/database/states.d.ts +1 -8
  7. package/packages/jam/database/states.d.ts.map +1 -1
  8. package/packages/jam/database/value-refs.d.ts +4 -5
  9. package/packages/jam/database/value-refs.d.ts.map +1 -1
  10. package/packages/jam/database-fjall/blocks.d.ts +1 -0
  11. package/packages/jam/database-fjall/blocks.d.ts.map +1 -1
  12. package/packages/jam/database-fjall/blocks.js +8 -3
  13. package/packages/jam/database-fjall/hybrid-states.d.ts +10 -4
  14. package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
  15. package/packages/jam/database-fjall/hybrid-states.js +22 -6
  16. package/packages/jam/database-fjall/root.d.ts +2 -0
  17. package/packages/jam/database-fjall/root.d.ts.map +1 -1
  18. package/packages/jam/database-fjall/root.js +8 -1
  19. package/packages/jam/database-fjall/root.test.d.ts +2 -0
  20. package/packages/jam/database-fjall/root.test.d.ts.map +1 -0
  21. package/packages/jam/database-fjall/root.test.js +23 -0
  22. package/packages/jam/database-fjall/states.d.ts +1 -0
  23. package/packages/jam/database-fjall/states.d.ts.map +1 -1
  24. package/packages/jam/database-fjall/states.js +6 -3
  25. package/packages/jam/node/export.d.ts.map +1 -1
  26. package/packages/jam/node/export.js +8 -17
  27. package/packages/jam/node/main-fuzz.d.ts.map +1 -1
  28. package/packages/jam/node/main-fuzz.js +51 -46
  29. package/packages/jam/node/main-importer.d.ts +6 -11
  30. package/packages/jam/node/main-importer.d.ts.map +1 -1
  31. package/packages/jam/node/main-importer.js +14 -28
  32. package/packages/jam/node/main.d.ts.map +1 -1
  33. package/packages/jam/node/main.js +6 -16
  34. package/packages/jam/rpc/test/e2e-server.d.ts.map +1 -1
  35. package/packages/jam/rpc/test/e2e-server.js +3 -5
  36. package/packages/jam/rpc/test/e2e.js +1 -1
  37. package/packages/jam/state-merkleization/index.d.ts +1 -1
  38. package/packages/jam/state-merkleization/index.js +1 -1
  39. package/packages/workers/api-node/config.d.ts +14 -46
  40. package/packages/workers/api-node/config.d.ts.map +1 -1
  41. package/packages/workers/api-node/config.js +31 -96
  42. package/packages/workers/api-node/config.test.js +82 -63
  43. package/packages/jam/database-lmdb/blocks.d.ts +0 -24
  44. package/packages/jam/database-lmdb/blocks.d.ts.map +0 -1
  45. package/packages/jam/database-lmdb/blocks.js +0 -82
  46. package/packages/jam/database-lmdb/hybrid-states.d.ts +0 -56
  47. package/packages/jam/database-lmdb/hybrid-states.d.ts.map +0 -1
  48. package/packages/jam/database-lmdb/hybrid-states.js +0 -149
  49. package/packages/jam/database-lmdb/hybrid-states.test.d.ts +0 -2
  50. package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map +0 -1
  51. package/packages/jam/database-lmdb/hybrid-states.test.js +0 -180
  52. package/packages/jam/database-lmdb/index.d.ts +0 -5
  53. package/packages/jam/database-lmdb/index.d.ts.map +0 -1
  54. package/packages/jam/database-lmdb/index.js +0 -4
  55. package/packages/jam/database-lmdb/root.d.ts +0 -26
  56. package/packages/jam/database-lmdb/root.d.ts.map +0 -1
  57. package/packages/jam/database-lmdb/root.js +0 -53
  58. package/packages/jam/database-lmdb/states.d.ts +0 -76
  59. package/packages/jam/database-lmdb/states.d.ts.map +0 -1
  60. package/packages/jam/database-lmdb/states.js +0 -145
  61. package/packages/jam/database-lmdb/states.test.d.ts +0 -2
  62. package/packages/jam/database-lmdb/states.test.d.ts.map +0 -1
  63. package/packages/jam/database-lmdb/states.test.js +0 -202
@@ -2,79 +2,11 @@ import { Decoder, Encoder } from "#@typeberry/codec";
2
2
  import { ChainSpec } from "#@typeberry/config";
3
3
  import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
4
4
  import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
5
- import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
6
5
  import { Blake2b } from "#@typeberry/hash";
7
6
  import { ThreadPort } from "./port.js";
8
- // Re-exported so the fuzz target can open one values session per run and reuse
9
- // it across resets (see `HybridWorkerConfig` / `mainFuzz`).
10
- export { FjallValuesSession };
11
- /**
12
- * Worker config for node.js, backed by the LMDB database.
13
- *
14
- * @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
15
- */
16
- export class LmdbWorkerConfig {
17
- nodeName;
18
- chainSpec;
19
- workerParams;
20
- dbPath;
21
- blake2b;
22
- ports;
23
- ephemeral;
24
- static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, }) {
25
- return new LmdbWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral);
26
- }
27
- /** Restore node config from a transferable config object. */
28
- static async fromTransferable(decodeParams, config) {
29
- if (config.databaseBackend !== "lmdb") {
30
- throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
31
- }
32
- const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
33
- return LmdbWorkerConfig.new({
34
- nodeName: config.nodeName,
35
- chainSpec,
36
- workerParams,
37
- dbPath: config.dbPath,
38
- blake2b,
39
- ports,
40
- });
41
- }
42
- constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
43
- // When set, the underlying database skips fsync. Only safe for throwaway
44
- // databases (the fuzz target wipes on reset). Not transferred to worker
45
- // threads, so the durable main node path always gets the default.
46
- ephemeral = false) {
47
- this.nodeName = nodeName;
48
- this.chainSpec = chainSpec;
49
- this.workerParams = workerParams;
50
- this.dbPath = dbPath;
51
- this.blake2b = blake2b;
52
- this.ports = ports;
53
- this.ephemeral = ephemeral;
54
- }
55
- async openDatabase(options = { readonly: true }) {
56
- const lmdb = LmdbRoot.new(this.dbPath, {
57
- readOnly: options.readonly,
58
- ephemeral: this.ephemeral,
59
- });
60
- return {
61
- getBlocksDb: () => LmdbBlocks.new(this.chainSpec, lmdb),
62
- getStatesDb: () => LmdbStates.new(this.chainSpec, this.blake2b, lmdb),
63
- close: async () => await lmdb.close(),
64
- };
65
- }
66
- /** Convert this config into a thread-transferable object. */
67
- intoTransferable(paramsCodec) {
68
- return {
69
- databaseBackend: "lmdb",
70
- nodeName: this.nodeName,
71
- chainSpec: this.chainSpec,
72
- workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
73
- dbPath: this.dbPath,
74
- workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
75
- };
76
- }
77
- }
7
+ // Re-exported so the fuzz target can open one keyspace per run and reuse it
8
+ // across resets.
9
+ export { FjallRoot, FjallValuesSession };
78
10
  /** Worker config for node.js, backed by a shared fjall engine. */
79
11
  export class FjallWorkerConfig {
80
12
  nodeName;
@@ -85,8 +17,9 @@ export class FjallWorkerConfig {
85
17
  ports;
86
18
  ephemeral;
87
19
  cacheSizeBytes;
88
- static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, }) {
89
- return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes);
20
+ sharedFjallKeyspace;
21
+ static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, sharedFjallKeyspace, }) {
22
+ return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes, sharedFjallKeyspace);
90
23
  }
91
24
  /** Restore node config from a transferable config object. */
92
25
  static async fromTransferable(decodeParams, config) {
@@ -106,7 +39,7 @@ export class FjallWorkerConfig {
106
39
  }
107
40
  constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
108
41
  // Kept for the fuzz/importer path. When set, persist() is skipped.
109
- ephemeral = false, cacheSizeBytes = undefined) {
42
+ ephemeral = false, cacheSizeBytes = undefined, sharedFjallKeyspace = undefined) {
110
43
  this.nodeName = nodeName;
111
44
  this.chainSpec = chainSpec;
112
45
  this.workerParams = workerParams;
@@ -115,13 +48,19 @@ export class FjallWorkerConfig {
115
48
  this.ports = ports;
116
49
  this.ephemeral = ephemeral;
117
50
  this.cacheSizeBytes = cacheSizeBytes;
51
+ this.sharedFjallKeyspace = sharedFjallKeyspace;
118
52
  }
119
53
  async openDatabase(options = { readonly: true }) {
120
- const fjall = await FjallRoot.open(this.dbPath, {
121
- readOnly: options.readonly,
122
- ephemeral: this.ephemeral,
123
- cacheSizeBytes: this.cacheSizeBytes,
124
- });
54
+ if (this.sharedFjallKeyspace !== undefined && options.readonly) {
55
+ throw new Error("Cannot open a read-only fjall database from a shared writable keyspace.");
56
+ }
57
+ const fjall = this.sharedFjallKeyspace ??
58
+ (await FjallRoot.open(this.dbPath, {
59
+ readOnly: options.readonly,
60
+ ephemeral: this.ephemeral,
61
+ cacheSizeBytes: this.cacheSizeBytes,
62
+ }));
63
+ const ownsFjall = this.sharedFjallKeyspace === undefined;
125
64
  let blocks = null;
126
65
  let states = null;
127
66
  try {
@@ -131,7 +70,9 @@ export class FjallWorkerConfig {
131
70
  ]);
132
71
  }
133
72
  catch (e) {
134
- await fjall.close();
73
+ if (ownsFjall) {
74
+ await fjall.close();
75
+ }
135
76
  throw e;
136
77
  }
137
78
  return {
@@ -150,7 +91,9 @@ export class FjallWorkerConfig {
150
91
  close: async () => {
151
92
  blocks = null;
152
93
  states = null;
153
- await fjall.close();
94
+ if (ownsFjall) {
95
+ await fjall.close();
96
+ }
154
97
  },
155
98
  };
156
99
  }
@@ -181,12 +124,7 @@ async function decodeTransferableConfig(decodeParams, config) {
181
124
  }
182
125
  /** Restore a persistent worker config from its transferable form. */
183
126
  export async function persistentConfigFromTransferable(decodeParams, config) {
184
- switch (config.databaseBackend) {
185
- case "lmdb":
186
- return LmdbWorkerConfig.fromTransferable(decodeParams, config);
187
- case "fjall":
188
- return FjallWorkerConfig.fromTransferable(decodeParams, config);
189
- }
127
+ return FjallWorkerConfig.fromTransferable(decodeParams, config);
190
128
  }
191
129
  /**
192
130
  * Collect the transferable objects (communication ports) embedded in a config.
@@ -232,8 +170,7 @@ export class InMemWorkerConfig {
232
170
  }
233
171
  /**
234
172
  * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
235
- * but large values persisted to disk. The `backend` picks where the values go
236
- * (lmdb or fjall).
173
+ * but large values persisted to disk via fjall.
237
174
  *
238
175
  * fjall opens its keyspace asynchronously, that is why `new` here is async.
239
176
  *
@@ -255,14 +192,12 @@ export class HybridWorkerConfig {
255
192
  ephemeral;
256
193
  compression;
257
194
  states;
258
- static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", sharedFjallSession, }) {
195
+ static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, sharedFjallKeyspace, }) {
259
196
  // The values store is created once here and shared across reopen. When a
260
- // session is given (fuzz reset reuse) we wrap it instead of opening a new one.
261
- const states = backend === "fjall"
262
- ? sharedFjallSession !== undefined
263
- ? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
264
- : await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
265
- : LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
197
+ // keyspace is given (fuzz reset reuse) we wrap it instead of opening a new one.
198
+ const states = sharedFjallKeyspace !== undefined
199
+ ? await FjallHybridSerializedStates.fromRoot(chainSpec, blake2b, sharedFjallKeyspace)
200
+ : await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral });
266
201
  return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
267
202
  }
268
203
  blocks;
@@ -1,46 +1,15 @@
1
1
  import assert from "node:assert";
2
2
  import * as fs from "node:fs";
3
3
  import { describe, it } from "node:test";
4
- import { MessageChannel } from "node:worker_threads";
5
4
  import { Bytes } from "#@typeberry/bytes";
6
5
  import { codec } from "#@typeberry/codec";
7
6
  import { tinyChainSpec } from "#@typeberry/config";
7
+ import { FjallRoot } from "#@typeberry/database-fjall";
8
8
  import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
9
9
  import { tryAsU32 } from "#@typeberry/numbers";
10
- import { configTransferList, FjallWorkerConfig, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
10
+ import { configTransferList, FjallWorkerConfig, HybridWorkerConfig } from "./config.js";
11
11
  import { ThreadPort } from "./port.js";
12
12
  const spec = tinyChainSpec;
13
- describe("LmdbWorkerConfig transfer list", () => {
14
- it("surfaces embedded worker ports so they can be transferred", async () => {
15
- const blake2b = await Blake2b.createHasher();
16
- const [portA, portB] = ThreadPort.pair(spec);
17
- const config = LmdbWorkerConfig.new({
18
- nodeName: "node",
19
- chainSpec: spec,
20
- workerParams: tryAsU32(7),
21
- dbPath: "db",
22
- blake2b,
23
- ports: new Map([["authorship-network", portA]]),
24
- });
25
- const transferable = config.intoTransferable(codec.varU32);
26
- const transferList = configTransferList(transferable);
27
- // the single embedded comms port must be reported for transfer
28
- assert.strictEqual(transferList.length, 1);
29
- const sink = new MessageChannel();
30
- try {
31
- // reproduces the bug: a config carrying a port cannot be cloned without
32
- // listing that port in the transfer list.
33
- assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
34
- // with the ports surfaced, posting succeeds.
35
- assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
36
- }
37
- finally {
38
- sink.port1.close();
39
- sink.port2.close();
40
- portB.close();
41
- }
42
- });
43
- });
44
13
  describe("FjallWorkerConfig transfer list", () => {
45
14
  it("surfaces embedded worker ports and marks the backend", async () => {
46
15
  const blake2b = await Blake2b.createHasher();
@@ -88,40 +57,90 @@ describe("FjallWorkerConfig transfer list", () => {
88
57
  fs.rmSync(dbPath, { recursive: true, force: true });
89
58
  }
90
59
  });
60
+ it("can borrow a shared fjall root without closing it", async () => {
61
+ const blake2b = await Blake2b.createHasher();
62
+ const dbPath = fs.mkdtempSync("typeberry-fjall-worker-shared-");
63
+ const root = await FjallRoot.open(dbPath, { ephemeral: true });
64
+ const config = FjallWorkerConfig.new({
65
+ nodeName: "node",
66
+ chainSpec: spec,
67
+ workerParams: undefined,
68
+ dbPath,
69
+ blake2b,
70
+ sharedFjallKeyspace: root,
71
+ });
72
+ try {
73
+ const db = await config.openDatabase({ readonly: false });
74
+ await db.close();
75
+ await root.deletePartition("headers");
76
+ const headers = await root.writablePartition("headers");
77
+ assert.strictEqual(headers.get(Bytes.zero(HASH_SIZE).raw), null);
78
+ }
79
+ finally {
80
+ await root.close();
81
+ fs.rmSync(dbPath, { recursive: true, force: true });
82
+ }
83
+ });
91
84
  });
92
85
  describe("HybridWorkerConfig", () => {
93
- // Both persistent backends must construct asynchronously and hand out a
94
- // working db. fjall is the experimental backend we want to benchmark.
95
- for (const backend of ["lmdb", "fjall"]) {
96
- it(`constructs and opens a ${backend}-backed hybrid db`, async () => {
97
- const blake2b = await Blake2b.createHasher();
98
- const dbPath = fs.mkdtempSync(`typeberry-hybrid-${backend}-`);
86
+ it("constructs and opens a fjall-backed hybrid db", async () => {
87
+ const blake2b = await Blake2b.createHasher();
88
+ const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-");
89
+ try {
90
+ const config = await HybridWorkerConfig.new({
91
+ nodeName: "node",
92
+ chainSpec: spec,
93
+ workerParams: undefined,
94
+ blake2b,
95
+ dbPath,
96
+ ephemeral: true,
97
+ });
98
+ const db = await config.openDatabase({ readonly: false });
99
+ const states = db.getStatesDb();
99
100
  try {
100
- const config = await HybridWorkerConfig.new({
101
- nodeName: "node",
102
- chainSpec: spec,
103
- workerParams: undefined,
104
- blake2b,
105
- dbPath,
106
- ephemeral: true,
107
- backend,
108
- });
109
- const db = await config.openDatabase({ readonly: false });
110
- const states = db.getStatesDb();
111
- try {
112
- assert.notStrictEqual(db.getBlocksDb(), undefined);
113
- assert.notStrictEqual(states, undefined);
114
- }
115
- finally {
116
- // The values store owns the on-disk resources (the no-op db.close()
117
- // does not), so close it explicitly to release the fjall keyspace.
118
- await states.close();
119
- await db.close();
120
- }
101
+ assert.notStrictEqual(db.getBlocksDb(), undefined);
102
+ assert.notStrictEqual(states, undefined);
121
103
  }
122
104
  finally {
123
- fs.rmSync(dbPath, { recursive: true, force: true });
105
+ // The values store owns the on-disk resources (the no-op db.close()
106
+ // does not), so close it explicitly to release the fjall keyspace.
107
+ await states.close();
108
+ await db.close();
124
109
  }
125
- });
126
- }
110
+ }
111
+ finally {
112
+ fs.rmSync(dbPath, { recursive: true, force: true });
113
+ }
114
+ });
115
+ it("can borrow a shared fjall root for the values store", async () => {
116
+ const blake2b = await Blake2b.createHasher();
117
+ const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-shared-");
118
+ const root = await FjallRoot.open(dbPath, { ephemeral: true });
119
+ try {
120
+ const config = await HybridWorkerConfig.new({
121
+ nodeName: "node",
122
+ chainSpec: spec,
123
+ workerParams: undefined,
124
+ blake2b,
125
+ dbPath,
126
+ ephemeral: true,
127
+ sharedFjallKeyspace: root,
128
+ });
129
+ const db = await config.openDatabase({ readonly: false });
130
+ const states = db.getStatesDb();
131
+ try {
132
+ assert.notStrictEqual(db.getBlocksDb(), undefined);
133
+ assert.notStrictEqual(states, undefined);
134
+ }
135
+ finally {
136
+ await states.close();
137
+ await db.close();
138
+ }
139
+ await root.deletePartition("values");
140
+ }
141
+ finally {
142
+ await root.close();
143
+ fs.rmSync(dbPath, { recursive: true, force: true });
144
+ }
145
+ });
127
146
  });
@@ -1,24 +0,0 @@
1
- import { type BlockView, type ExtrinsicView, type HeaderHash, type HeaderView, type StateRootHash } from "#@typeberry/block";
2
- import type { ChainSpec } from "#@typeberry/config";
3
- import type { BlocksDb } from "#@typeberry/database/blocks.js";
4
- import { type WithHash } from "#@typeberry/hash";
5
- import type { LmdbRoot } from "./root.js";
6
- export declare class LmdbBlocks implements BlocksDb {
7
- private readonly chainSpec;
8
- private readonly root;
9
- private readonly extrinsics;
10
- private readonly headers;
11
- private readonly postStateRoots;
12
- static new(chainSpec: ChainSpec, root: LmdbRoot): LmdbBlocks;
13
- private constructor();
14
- setPostStateRoot(hash: HeaderHash, postStateRoot: StateRootHash): Promise<void>;
15
- getPostStateRoot(hash: HeaderHash): StateRootHash | null;
16
- insertBlock(block: WithHash<HeaderHash, BlockView>): Promise<void>;
17
- setBestHeaderHash(hash: HeaderHash): Promise<void>;
18
- getBestHeaderHash(): HeaderHash;
19
- getHeader(hash: HeaderHash): HeaderView | null;
20
- getExtrinsic(hash: HeaderHash): ExtrinsicView | null;
21
- markUnused(hash: HeaderHash): void;
22
- close(): Promise<void>;
23
- }
24
- //# sourceMappingURL=blocks.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAQjD,qBAAa,UAAW,YAAW,QAAQ;IAUvC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAEvC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ;IAI/C,OAAO;IAeD,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrF,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQlD,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IASlE,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxD,iBAAiB,IAAI,UAAU;IAS/B,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAS9C,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQpD,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAM5B,KAAK;CAGZ"}
@@ -1,82 +0,0 @@
1
- import { Extrinsic, Header, } from "#@typeberry/block";
2
- import { Bytes } from "#@typeberry/bytes";
3
- import { Decoder } from "#@typeberry/codec";
4
- import { HASH_SIZE } from "#@typeberry/hash";
5
- const BEST_BLOCK = "best hash and posterior state root";
6
- // TODO [ToDr] consider having a changeset for transactions,
7
- // where we store all `insert ++ key ++ value` and `remove ++ key`
8
- // in a single `Uint8Array` JAM-encoded. That could then
9
- // be efficiently transferred between threads.
10
- export class LmdbBlocks {
11
- chainSpec;
12
- root;
13
- extrinsics;
14
- headers;
15
- postStateRoots;
16
- static new(chainSpec, root) {
17
- return new LmdbBlocks(chainSpec, root);
18
- }
19
- constructor(chainSpec, root) {
20
- this.chainSpec = chainSpec;
21
- this.root = root;
22
- // NOTE [ToDr] Extrinsics are stored under header hash, not their hash. Revise if it's an issue.
23
- this.extrinsics = this.root.subDb("extrinsics");
24
- this.headers = this.root.subDb("headers");
25
- // NOTE [ToDr] We currently store all posterior state roots, however it's
26
- // most likely very redundant. We probably only need to store the posterior
27
- // state roots of recent blocks to be able to quickly resolve forks
28
- // OR we need a way to be able to traverse the blocks history forward
29
- // (i.e. know what next block(s) is).
30
- this.postStateRoots = this.root.subDb("postStateRoots");
31
- }
32
- async setPostStateRoot(hash, postStateRoot) {
33
- await this.postStateRoots.put(hash.raw, postStateRoot.raw);
34
- }
35
- getPostStateRoot(hash) {
36
- const postStateRoot = this.postStateRoots.get(hash.raw);
37
- if (postStateRoot === undefined) {
38
- return null;
39
- }
40
- return Bytes.fromBlob(postStateRoot, HASH_SIZE).asOpaque();
41
- }
42
- async insertBlock(block) {
43
- const header = block.data.header.view().encoded();
44
- const extrinsic = block.data.extrinsic.view().encoded();
45
- await this.root.db.transaction(() => {
46
- this.headers.put(block.hash.raw, header.raw);
47
- this.extrinsics.put(block.hash.raw, extrinsic.raw);
48
- });
49
- }
50
- async setBestHeaderHash(hash) {
51
- await this.root.db.put(BEST_BLOCK, hash.raw);
52
- }
53
- getBestHeaderHash() {
54
- const bestHeaderHash = this.root.db.get(BEST_BLOCK);
55
- if (bestHeaderHash === undefined) {
56
- return Bytes.zero(HASH_SIZE).asOpaque();
57
- }
58
- return Bytes.fromBlob(bestHeaderHash, HASH_SIZE).asOpaque();
59
- }
60
- getHeader(hash) {
61
- const data = this.headers.get(hash.raw);
62
- if (data === undefined) {
63
- return null;
64
- }
65
- return Decoder.decodeObject(Header.Codec.View, data, this.chainSpec);
66
- }
67
- getExtrinsic(hash) {
68
- const data = this.extrinsics.get(hash.raw);
69
- if (data === undefined) {
70
- return null;
71
- }
72
- return Decoder.decodeObject(Extrinsic.Codec.View, data, this.chainSpec);
73
- }
74
- markUnused(hash) {
75
- this.headers.removeSync(hash.raw);
76
- this.extrinsics.removeSync(hash.raw);
77
- this.postStateRoots.removeSync(hash.raw);
78
- }
79
- async close() {
80
- await Promise.all([this.headers.close(), this.extrinsics.close(), this.postStateRoots.close()]);
81
- }
82
- }
@@ -1,56 +0,0 @@
1
- import type { HeaderHash, StateRootHash } from "#@typeberry/block";
2
- import type { ChainSpec } from "#@typeberry/config";
3
- import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
4
- import type { Blake2b } from "#@typeberry/hash";
5
- import type { ServicesUpdate, State } from "#@typeberry/state";
6
- import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
7
- import { OK, Result } from "#@typeberry/utils";
8
- /**
9
- * Hybrid serialized-states db.
10
- *
11
- * States (leafs) are kept in-memory, but large values are persisted to lmdb.
12
- * Reads go straight to lmdb, which keeps its own page cache.
13
- * NOTE: this DB is designed for long fuzzing and to be used with pruning to
14
- * keep the heap usage bounded.
15
- *
16
- * Values that no longer belong to any surviving state are removed from lmdb,
17
- * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
18
- * finality signal. Counts are not persisted: this db cannot resume from disk
19
- * anyway (the leaf sets live in memory), so values left over by a previous
20
- * run are never collected.
21
- */
22
- export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
23
- private readonly spec;
24
- private readonly blake2b;
25
- private readonly root;
26
- private readonly inMemStates;
27
- private readonly lmdbValues;
28
- private readonly valuesDb;
29
- private readonly refsStore;
30
- private readonly refs;
31
- private pendingCleanup;
32
- static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
33
- spec: ChainSpec;
34
- blake2b: Blake2b;
35
- dbPath: string;
36
- readOnly?: boolean;
37
- ephemeral?: boolean;
38
- compression?: boolean;
39
- }): HybridSerializedStates;
40
- private constructor();
41
- insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
42
- updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
43
- getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
44
- getState(header: HeaderHash): SerializedState<LeafDb> | null;
45
- commitFinalized(headers: HeaderHash[]): void;
46
- markUnused(header: HeaderHash): void;
47
- /** Apply a refcounting update and remove values that lost their last reference. */
48
- private applyRefs;
49
- diskSizeInBytes(): number | null;
50
- close(): Promise<void>;
51
- /** Write new large values to LMDB in one transaction. */
52
- private writeValues;
53
- /** Read a value from LMDB. */
54
- private readValue;
55
- }
56
- //# sourceMappingURL=hybrid-states.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}