@typeberry/lib 0.9.0-3f2c45b → 0.9.0-4a9f363

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/package.json +1 -1
  2. package/packages/core/collections/sorted-array.d.ts +1 -1
  3. package/packages/core/collections/sorted-array.d.ts.map +1 -1
  4. package/packages/core/collections/sorted-array.js +4 -2
  5. package/packages/core/collections/sorted-array.test.js +10 -0
  6. package/packages/core/collections/sorted-set.d.ts +1 -1
  7. package/packages/core/collections/sorted-set.d.ts.map +1 -1
  8. package/packages/core/collections/sorted-set.js +6 -2
  9. package/packages/core/collections/sorted-set.test.js +24 -0
  10. package/packages/core/utils/index.d.ts +1 -0
  11. package/packages/core/utils/index.d.ts.map +1 -1
  12. package/packages/core/utils/index.js +1 -0
  13. package/packages/core/utils/shutdown.d.ts +18 -0
  14. package/packages/core/utils/shutdown.d.ts.map +1 -0
  15. package/packages/core/utils/shutdown.js +3 -0
  16. package/packages/core/utils/shutdown.node.d.ts +5 -0
  17. package/packages/core/utils/shutdown.node.d.ts.map +1 -0
  18. package/packages/core/utils/shutdown.node.js +76 -0
  19. package/packages/core/utils/shutdown.test.d.ts +2 -0
  20. package/packages/core/utils/shutdown.test.d.ts.map +1 -0
  21. package/packages/core/utils/shutdown.test.js +164 -0
  22. package/packages/extensions/ipc/server.d.ts.map +1 -1
  23. package/packages/extensions/ipc/server.js +7 -0
  24. package/packages/jam/config-node/node-config.d.ts +20 -2
  25. package/packages/jam/config-node/node-config.d.ts.map +1 -1
  26. package/packages/jam/config-node/node-config.js +25 -3
  27. package/packages/jam/config-node/node-config.test.js +8 -1
  28. package/packages/jam/database/index.d.ts +1 -0
  29. package/packages/jam/database/index.d.ts.map +1 -1
  30. package/packages/jam/database/index.js +1 -0
  31. package/packages/jam/database/leaf-db-update.d.ts +1 -0
  32. package/packages/jam/database/leaf-db-update.d.ts.map +1 -1
  33. package/packages/jam/database/leaf-db-update.js +22 -5
  34. package/packages/jam/database/leaf-db-update.test.d.ts +2 -0
  35. package/packages/jam/database/leaf-db-update.test.d.ts.map +1 -0
  36. package/packages/jam/database/leaf-db-update.test.js +81 -0
  37. package/packages/jam/database/serialized-states-db.d.ts +1 -0
  38. package/packages/jam/database/serialized-states-db.d.ts.map +1 -1
  39. package/packages/jam/database/serialized-states-db.js +4 -0
  40. package/packages/jam/database/states.d.ts +11 -0
  41. package/packages/jam/database/states.d.ts.map +1 -1
  42. package/packages/jam/database/states.js +3 -0
  43. package/packages/jam/database/value-refs.d.ts +117 -0
  44. package/packages/jam/database/value-refs.d.ts.map +1 -0
  45. package/packages/jam/database/value-refs.js +206 -0
  46. package/packages/jam/database/value-refs.test.d.ts +2 -0
  47. package/packages/jam/database/value-refs.test.d.ts.map +1 -0
  48. package/packages/jam/database/value-refs.test.js +165 -0
  49. package/packages/jam/database-fjall/blocks.d.ts +25 -0
  50. package/packages/jam/database-fjall/blocks.d.ts.map +1 -0
  51. package/packages/jam/database-fjall/blocks.js +86 -0
  52. package/packages/jam/database-fjall/blocks.test.d.ts +2 -0
  53. package/packages/jam/database-fjall/blocks.test.d.ts.map +1 -0
  54. package/packages/jam/database-fjall/blocks.test.js +62 -0
  55. package/packages/jam/database-fjall/hybrid-states.d.ts +14 -0
  56. package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
  57. package/packages/jam/database-fjall/hybrid-states.js +55 -6
  58. package/packages/jam/database-fjall/index.d.ts +2 -0
  59. package/packages/jam/database-fjall/index.d.ts.map +1 -1
  60. package/packages/jam/database-fjall/index.js +2 -0
  61. package/packages/jam/database-fjall/root.d.ts +14 -8
  62. package/packages/jam/database-fjall/root.d.ts.map +1 -1
  63. package/packages/jam/database-fjall/root.js +25 -9
  64. package/packages/jam/database-fjall/states.d.ts +31 -0
  65. package/packages/jam/database-fjall/states.d.ts.map +1 -0
  66. package/packages/jam/database-fjall/states.js +99 -0
  67. package/packages/jam/database-fjall/states.test.d.ts +2 -0
  68. package/packages/jam/database-fjall/states.test.d.ts.map +1 -0
  69. package/packages/jam/database-fjall/states.test.js +150 -0
  70. package/packages/jam/database-lmdb/hybrid-states.d.ts +12 -0
  71. package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
  72. package/packages/jam/database-lmdb/hybrid-states.js +45 -4
  73. package/packages/jam/database-lmdb/states.d.ts +1 -0
  74. package/packages/jam/database-lmdb/states.d.ts.map +1 -1
  75. package/packages/jam/database-lmdb/states.js +5 -0
  76. package/packages/jam/node/export.d.ts.map +1 -1
  77. package/packages/jam/node/export.js +18 -9
  78. package/packages/jam/node/main-fuzz.d.ts +4 -1
  79. package/packages/jam/node/main-fuzz.d.ts.map +1 -1
  80. package/packages/jam/node/main-fuzz.js +111 -82
  81. package/packages/jam/node/main-importer.d.ts +4 -2
  82. package/packages/jam/node/main-importer.d.ts.map +1 -1
  83. package/packages/jam/node/main-importer.js +26 -12
  84. package/packages/jam/node/main.d.ts.map +1 -1
  85. package/packages/jam/node/main.js +42 -15
  86. package/packages/jam/node/workers.d.ts +4 -4
  87. package/packages/jam/node/workers.d.ts.map +1 -1
  88. package/packages/workers/api/config.d.ts +2 -2
  89. package/packages/workers/api/config.d.ts.map +1 -1
  90. package/packages/workers/api/config.js +1 -1
  91. package/packages/workers/api/port.d.ts +5 -1
  92. package/packages/workers/api/port.d.ts.map +1 -1
  93. package/packages/workers/api/port.js +56 -13
  94. package/packages/workers/api/port.test.js +35 -0
  95. package/packages/workers/api-node/config.d.ts +45 -4
  96. package/packages/workers/api-node/config.d.ts.map +1 -1
  97. package/packages/workers/api-node/config.js +127 -9
  98. package/packages/workers/api-node/config.test.js +52 -3
  99. package/packages/workers/api-node/port.d.ts +3 -0
  100. package/packages/workers/api-node/port.d.ts.map +1 -1
  101. package/packages/workers/api-node/port.js +31 -2
  102. package/packages/workers/api-node/port.test.js +17 -0
  103. package/packages/workers/api-node/protocol.d.ts +3 -3
  104. package/packages/workers/api-node/protocol.d.ts.map +1 -1
  105. package/packages/workers/api-node/protocol.js +18 -12
  106. package/packages/workers/block-authorship/main.js +1 -1
  107. package/packages/workers/importer/main.d.ts +1 -1
  108. package/packages/workers/importer/main.d.ts.map +1 -1
  109. package/packages/workers/importer/main.js +1 -1
  110. package/packages/workers/jam-network/main.js +1 -1
@@ -1,6 +1,7 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
- import { open } from "@fjall-js/fjall";
3
+ import { open, openReadonly, } from "@fjall-js/fjall";
4
+ const DEFAULT_CACHE_SIZE_BYTES = 64 * 1024 * 1024;
4
5
  /**
5
6
  * Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
6
7
  */
@@ -29,19 +30,31 @@ export class FjallRoot {
29
30
  this.options = options;
30
31
  }
31
32
  /** Open (or create) a fjall keyspace at the given path. */
32
- static async open(dbPath, options) {
33
- // Forward our options to the binding: `ephemeral` makes `close()` skip the
34
- // sync-all fsync, `cacheSizeBytes` bounds how much we keep in memory.
35
- const keyspace = await open(dbPath, {
36
- ephemeral: options.ephemeral,
37
- cacheSizeBytes: options.cacheSizeBytes,
38
- });
33
+ static async open(dbPath, options = {}) {
34
+ // fjall-js 0.3 shares one engine per path. Readers must use the read-only
35
+ // wrapper surface; durability is driven explicitly through persist().
36
+ const config = {
37
+ path: dbPath,
38
+ cacheSizeBytes: options.cacheSizeBytes ?? DEFAULT_CACHE_SIZE_BYTES,
39
+ };
40
+ const keyspace = options.readOnly === true ? await openReadonly(config) : await open(config);
39
41
  return new FjallRoot(keyspace, dbPath, options);
40
42
  }
43
+ /** Whether this root was opened through fjall's read-only surface. */
44
+ get readOnly() {
45
+ return this.options.readOnly === true;
46
+ }
41
47
  /** Open (or create) a partition under this keyspace. */
42
48
  async partition(name) {
43
49
  return this.keyspace.partition(name);
44
50
  }
51
+ /** Open a writable partition, failing early when this root is read-only. */
52
+ async writablePartition(name) {
53
+ if (this.readOnly) {
54
+ throw new Error(`Cannot open writable fjall partition '${name}' from a read-only keyspace.`);
55
+ }
56
+ return (await this.keyspace.partition(name));
57
+ }
45
58
  /**
46
59
  * Flush the journal to disk so prior writes survive a crash.
47
60
  *
@@ -52,6 +65,9 @@ export class FjallRoot {
52
65
  if (this.options.ephemeral === true) {
53
66
  return;
54
67
  }
68
+ if (this.readOnly) {
69
+ throw new Error("Cannot persist a read-only fjall keyspace.");
70
+ }
55
71
  await this.keyspace.persist();
56
72
  }
57
73
  /**
@@ -69,7 +85,7 @@ export class FjallRoot {
69
85
  return null;
70
86
  }
71
87
  }
72
- /** Persist with `sync-all` and release the keyspace handle. */
88
+ /** Release this keyspace handle. Call persist() first when durability is needed. */
73
89
  async close() {
74
90
  await this.keyspace.close();
75
91
  }
@@ -0,0 +1,31 @@
1
+ import type { HeaderHash, StateRootHash } from "#@typeberry/block";
2
+ import type { ChainSpec } from "#@typeberry/config";
3
+ import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
4
+ import type { Blake2b } from "#@typeberry/hash";
5
+ import type { ServicesUpdate, State } from "#@typeberry/state";
6
+ import type { StateEntries } from "#@typeberry/state-merkleization";
7
+ import { SerializedState } from "#@typeberry/state-merkleization";
8
+ import { OK, Result } from "#@typeberry/utils";
9
+ import { type FjallRoot } from "./root.js";
10
+ /** fjall-backed full on-disk serialized state storage. */
11
+ export declare class FjallStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
12
+ private readonly spec;
13
+ private readonly blake2b;
14
+ private readonly root;
15
+ private readonly states;
16
+ private readonly values;
17
+ static open(spec: ChainSpec, blake2b: Blake2b, root: FjallRoot): Promise<FjallStates>;
18
+ private readonly valuesDb;
19
+ private constructor();
20
+ insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
21
+ updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
22
+ getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
23
+ getState(headerHash: HeaderHash): SerializedState<LeafDb> | null;
24
+ commitFinalized(_headers: HeaderHash[]): void;
25
+ markUnused(headerHash: HeaderHash): void;
26
+ diskSizeInBytes(): number | null;
27
+ close(): Promise<void>;
28
+ private updateAndCommit;
29
+ private readValue;
30
+ }
31
+ //# sourceMappingURL=states.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAE/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAI9F,0DAA0D;AAC1D,qBAAa,WAAY,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS7F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;WAZZ,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAK3F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,OAAO;IAUD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAQxG,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAUlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAahE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAK7C,UAAU,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMxC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;YAEG,eAAe;IAyB7B,OAAO,CAAC,SAAS;CAOlB"}
@@ -0,0 +1,99 @@
1
+ import { BytesBlob } from "#@typeberry/bytes";
2
+ import { SortedSet } from "#@typeberry/collections";
3
+ import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
4
+ import { Logger } from "#@typeberry/logger";
5
+ import { SerializedState, StateEntryUpdateAction, serializeStateUpdate } from "#@typeberry/state-merkleization";
6
+ import { leafComparator } from "#@typeberry/trie";
7
+ import { OK, Result, resultToString } from "#@typeberry/utils";
8
+ import { toUint8Array } from "./root.js";
9
+ const logger = Logger.new(import.meta.filename, "db");
10
+ /** fjall-backed full on-disk serialized state storage. */
11
+ export class FjallStates {
12
+ spec;
13
+ blake2b;
14
+ root;
15
+ states;
16
+ values;
17
+ static async open(spec, blake2b, root) {
18
+ const [states, values] = await Promise.all([root.partition("states"), root.partition("values")]);
19
+ return new FjallStates(spec, blake2b, root, states, values);
20
+ }
21
+ valuesDb;
22
+ constructor(spec, blake2b, root, states, values) {
23
+ this.spec = spec;
24
+ this.blake2b = blake2b;
25
+ this.root = root;
26
+ this.states = states;
27
+ this.values = values;
28
+ this.valuesDb = { get: (key) => this.readValue(key) };
29
+ }
30
+ async insertInitialState(headerHash, entries) {
31
+ return await this.updateAndCommit(headerHash, SortedSet.fromArray(leafComparator, []), Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
32
+ }
33
+ async updateAndSetState(headerHash, state, update) {
34
+ const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
35
+ const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
36
+ const res = await this.updateAndCommit(headerHash, newLeafs, updatedValues);
37
+ if (res.isOk) {
38
+ state.updateBackend(LeafDb.fromLeaves(newLeafs, this.valuesDb));
39
+ }
40
+ return res;
41
+ }
42
+ async getStateRoot(state) {
43
+ return state.backend.getStateRoot(this.blake2b);
44
+ }
45
+ getState(headerHash) {
46
+ const leafNodes = toUint8Array(this.states.get(headerHash.raw));
47
+ if (leafNodes === null) {
48
+ return null;
49
+ }
50
+ const leafDbResult = LeafDb.fromLeavesBlob(BytesBlob.blobFrom(leafNodes), this.valuesDb);
51
+ if (leafDbResult.isError) {
52
+ throw new Error(`Inconsistent DB. Invalid leaf nodes for ${headerHash}: ${resultToString(leafDbResult)}`);
53
+ }
54
+ return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
55
+ }
56
+ commitFinalized(_headers) {
57
+ // Values are never pruned here. This db survives restarts, so refcounting
58
+ // would need counts persisted (and crash-consistent) alongside the values.
59
+ }
60
+ markUnused(headerHash) {
61
+ void writable(this.states, this.root)
62
+ .remove(headerHash.raw)
63
+ .catch((e) => logger.warn `Failed to prune state ${headerHash}: ${e}`);
64
+ }
65
+ diskSizeInBytes() {
66
+ return this.root.sizeInBytes();
67
+ }
68
+ async close() { }
69
+ async updateAndCommit(headerHash, leafs, data) {
70
+ const { values } = updateLeafs(leafs, this.blake2b, data);
71
+ const stateLeafs = BytesBlob.blobFromParts(leafs.array.map((x) => x.node.raw));
72
+ try {
73
+ // Preserve dependency order: values first, then leaves that may reference them.
74
+ if (values.length > 0) {
75
+ await writable(this.values, this.root).insertBatch(values.map(([hash, val]) => ({ key: hash.raw, value: val.raw })));
76
+ }
77
+ await writable(this.states, this.root).insert(headerHash.raw, stateLeafs.raw);
78
+ await this.root.persist();
79
+ }
80
+ catch (e) {
81
+ logger.error `${e}`;
82
+ return Result.error(StateUpdateError.Commit, () => `Failed to commit state update: ${e}`);
83
+ }
84
+ return Result.ok(OK);
85
+ }
86
+ readValue(key) {
87
+ const val = toUint8Array(this.values.get(key.raw));
88
+ if (val === null) {
89
+ throw new Error(`Missing required value: ${key} in the DB`);
90
+ }
91
+ return val;
92
+ }
93
+ }
94
+ function writable(partition, root) {
95
+ if (root.readOnly) {
96
+ throw new Error("Cannot write through a read-only fjall partition.");
97
+ }
98
+ return partition;
99
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=states.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,150 @@
1
+ import assert from "node:assert";
2
+ import * as fs from "node:fs";
3
+ import { afterEach, before, beforeEach, describe, it } from "node:test";
4
+ import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
5
+ import { Bytes, BytesBlob } from "#@typeberry/bytes";
6
+ import { tinyChainSpec } from "#@typeberry/config";
7
+ import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
8
+ import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
9
+ import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
10
+ import { testState } from "#@typeberry/state/test.utils.js";
11
+ import { StateEntries } from "#@typeberry/state-merkleization";
12
+ import { deepEqual, OK, Result } from "#@typeberry/utils";
13
+ import { FjallRoot } from "./root.js";
14
+ import { FjallStates } from "./states.js";
15
+ let blake2b;
16
+ before(async () => {
17
+ blake2b = await Blake2b.createHasher();
18
+ });
19
+ describe("Fjall states database", () => {
20
+ const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
21
+ const spec = tinyChainSpec;
22
+ let tmpDir = "";
23
+ beforeEach(() => {
24
+ tmpDir = fs.mkdtempSync("typeberry-fjall-states-");
25
+ });
26
+ afterEach(() => {
27
+ fs.rmSync(tmpDir, { recursive: true, force: true });
28
+ });
29
+ it("imports and reads an empty state", async () => {
30
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
31
+ const states = await FjallStates.open(spec, blake2b, root);
32
+ try {
33
+ const emptyState = InMemoryState.empty(spec);
34
+ const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
35
+ const emptyRoot = serialized.getRootHash(blake2b);
36
+ deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
37
+ const newState = states.getState(headerHash);
38
+ assert.ok(newState !== null);
39
+ assert.strictEqual(`${await states.getStateRoot(newState)}`, `${emptyRoot}`);
40
+ deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
41
+ }
42
+ finally {
43
+ await states.close();
44
+ await root.close();
45
+ }
46
+ });
47
+ it("reads large values back from disk", async () => {
48
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
49
+ const states = await FjallStates.open(spec, blake2b, root);
50
+ try {
51
+ const big1 = BytesBlob.blobFromString("x".repeat(100));
52
+ const big2 = BytesBlob.blobFromString("y".repeat(100));
53
+ const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
54
+ const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
55
+ const entries = StateEntries.fromEntriesUnsafe([
56
+ [key1, big1],
57
+ [key2, big2],
58
+ ]);
59
+ deepEqual(await states.insertInitialState(headerHash, entries), Result.ok(OK));
60
+ const state = states.getState(headerHash);
61
+ assert.ok(state !== null);
62
+ assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
63
+ assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
64
+ }
65
+ finally {
66
+ await states.close();
67
+ await root.close();
68
+ }
69
+ });
70
+ it("updates state", async () => {
71
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
72
+ const states = await FjallStates.open(spec, blake2b, root);
73
+ try {
74
+ const state = InMemoryState.empty(spec);
75
+ await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
76
+ const newState = states.getState(headerHash);
77
+ assert.ok(newState !== null);
78
+ const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
79
+ const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
80
+ const stateUpdate = {
81
+ timeslot: tryAsTimeSlot(15),
82
+ privilegedServices: PrivilegedServices.create({
83
+ manager: tryAsServiceId(1),
84
+ assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
85
+ delegator: tryAsServiceId(3),
86
+ registrar: tryAsServiceId(4),
87
+ autoAccumulateServices: new Map(),
88
+ }),
89
+ updated: new Map([
90
+ [
91
+ tryAsServiceId(1),
92
+ UpdateService.create({
93
+ serviceInfo: ServiceAccountInfo.create({
94
+ codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
95
+ balance: tryAsU64(1_000_000),
96
+ accumulateMinGas: tryAsServiceGas(10_000),
97
+ onTransferMinGas: tryAsServiceGas(5_000),
98
+ storageUtilisationBytes: tryAsU64(1_000),
99
+ gratisStorage: tryAsU64(0),
100
+ storageUtilisationCount: tryAsU32(1),
101
+ created: tryAsTimeSlot(0),
102
+ lastAccumulation: tryAsTimeSlot(0),
103
+ parentService: tryAsServiceId(0),
104
+ }),
105
+ lookupHistory,
106
+ }),
107
+ ],
108
+ ]),
109
+ };
110
+ deepEqual(state.applyUpdate(stateUpdate), Result.ok(OK));
111
+ deepEqual(await states.updateAndSetState(headerHash2, newState, stateUpdate), Result.ok(OK));
112
+ const updatedState = states.getState(headerHash2);
113
+ assert.ok(updatedState !== null);
114
+ deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
115
+ [
116
+ tryAsServiceId(1),
117
+ {
118
+ storageKeys: [],
119
+ preimages: [],
120
+ lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
121
+ },
122
+ ],
123
+ ])), state);
124
+ assert.strictEqual(`${await states.getStateRoot(updatedState)}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
125
+ }
126
+ finally {
127
+ await states.close();
128
+ await root.close();
129
+ }
130
+ });
131
+ it("imports a more complex state", async () => {
132
+ const root = await FjallRoot.open(tmpDir, { ephemeral: true });
133
+ const states = await FjallStates.open(spec, blake2b, root);
134
+ try {
135
+ const initialState = testState();
136
+ const initialService = initialState.services.get(tryAsServiceId(0));
137
+ assert.ok(initialService !== undefined);
138
+ const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
139
+ deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
140
+ const newState = states.getState(headerHash);
141
+ assert.ok(newState !== null);
142
+ assert.strictEqual(`${await states.getStateRoot(newState)}`, `${serialized.getRootHash(blake2b)}`);
143
+ deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
144
+ }
145
+ finally {
146
+ await states.close();
147
+ await root.close();
148
+ }
149
+ });
150
+ });
@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
12
12
  * Reads go straight to lmdb, which keeps its own page cache.
13
13
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
14
14
  * keep the heap usage bounded.
15
+ *
16
+ * Values that no longer belong to any surviving state are removed from lmdb,
17
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
18
+ * finality signal. Counts are not persisted: this db cannot resume from disk
19
+ * anyway (the leaf sets live in memory), so values left over by a previous
20
+ * run are never collected.
15
21
  */
16
22
  export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
17
23
  private readonly spec;
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
20
26
  private readonly inMemStates;
21
27
  private readonly lmdbValues;
22
28
  private readonly valuesDb;
29
+ private readonly refsStore;
30
+ private readonly refs;
31
+ private pendingCleanup;
23
32
  static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
24
33
  spec: ChainSpec;
25
34
  blake2b: Blake2b;
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
33
42
  updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
34
43
  getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
35
44
  getState(header: HeaderHash): SerializedState<LeafDb> | null;
45
+ commitFinalized(headers: HeaderHash[]): void;
36
46
  markUnused(header: HeaderHash): void;
47
+ /** Apply a refcounting update and remove values that lost their last reference. */
48
+ private applyRefs;
37
49
  diskSizeInBytes(): number | null;
38
50
  close(): Promise<void>;
39
51
  /** Write new large values to LMDB in one transaction. */
@@ -1 +1 @@
1
- {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
1
+ {"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
@@ -1,9 +1,11 @@
1
1
  import { HashDictionary, SortedSet } from "#@typeberry/collections";
2
- import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
2
+ import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
3
+ import { Logger } from "#@typeberry/logger";
3
4
  import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
4
5
  import { leafComparator } from "#@typeberry/trie";
5
6
  import { OK, Result } from "#@typeberry/utils";
6
7
  import { LmdbRoot } from "./root.js";
8
+ const logger = Logger.new(import.meta.filename, "db");
7
9
  /**
8
10
  * Hybrid serialized-states db.
9
11
  *
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
11
13
  * Reads go straight to lmdb, which keeps its own page cache.
12
14
  * NOTE: this DB is designed for long fuzzing and to be used with pruning to
13
15
  * keep the heap usage bounded.
16
+ *
17
+ * Values that no longer belong to any surviving state are removed from lmdb,
18
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
19
+ * finality signal. Counts are not persisted: this db cannot resume from disk
20
+ * anyway (the leaf sets live in memory), so values left over by a previous
21
+ * run are never collected.
14
22
  */
15
23
  export class HybridSerializedStates {
16
24
  spec;
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
20
28
  lmdbValues;
21
29
  // A single shared values accessor reused by every `LeafDb` we hand out.
22
30
  valuesDb;
31
+ refsStore = new InMemoryValueRefsStore();
32
+ refs = new ValueRefs(this.refsStore);
33
+ // Queue of not-yet-committed value removals, awaited on close.
34
+ pendingCleanup = Promise.resolve();
23
35
  static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
24
36
  const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
25
37
  return new HybridSerializedStates(spec, blake2b, root);
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
38
50
  return res;
39
51
  }
40
52
  this.inMemStates.set(headerHash, leafs);
53
+ this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
41
54
  return Result.ok(OK);
42
55
  }
43
56
  async updateAndSetState(header, state, update) {
44
57
  const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
45
58
  // Clone the leaf set before mutating: the previous state keeps using its own.
46
59
  const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
47
- const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
60
+ const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
48
61
  const res = await this.writeValues(values);
49
62
  if (res.isError) {
50
63
  // Leave the caller's state untouched: its new leaves would reference
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
55
68
  // values are durably written.
56
69
  state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
57
70
  this.inMemStates.set(header, leafs);
71
+ this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
58
72
  return Result.ok(OK);
59
73
  }
60
74
  async getStateRoot(state) {
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
68
82
  const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
69
83
  return SerializedState.new(this.spec, this.blake2b, leafDb);
70
84
  }
85
+ commitFinalized(headers) {
86
+ this.applyRefs(this.refs.commitFinalized(headers));
87
+ }
71
88
  markUnused(header) {
72
- // We only remove the state from memory - values are not pruned at all,
73
- // but since they are stored on disk we should be safe.
89
+ // Release the speculative references first (a no-op for finalized states,
90
+ // whose deltas were already consumed by `commitFinalized`).
91
+ this.applyRefs(this.refs.releaseUnfinalized(header));
74
92
  this.inMemStates.delete(header);
75
93
  }
94
+ /** Apply a refcounting update and remove values that lost their last reference. */
95
+ applyRefs(update) {
96
+ this.refsStore.apply(update);
97
+ if (update.removeValues.length === 0) {
98
+ return;
99
+ }
100
+ // Queued, not awaited: a failed removal only leaks a value.
101
+ this.pendingCleanup = this.pendingCleanup
102
+ .then(() => this.lmdbValues.transaction(() => {
103
+ for (const v of update.removeValues) {
104
+ this.lmdbValues.remove(v.raw);
105
+ }
106
+ }))
107
+ .catch((e) => {
108
+ logger.error `Failed to remove unreferenced values: ${e}`;
109
+ });
110
+ }
76
111
  diskSizeInBytes() {
77
112
  return this.root.sizeInBytes();
78
113
  }
79
114
  async close() {
115
+ await this.pendingCleanup;
80
116
  await this.lmdbValues.close();
81
117
  await this.root.close();
82
118
  }
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
86
122
  return Result.ok(OK);
87
123
  }
88
124
  try {
125
+ // Flush queued removals first: a pending cleanup might be about to delete
126
+ // a content hash we are re-inserting now. Writing behind it keeps the
127
+ // re-inserted value on disk (removals are only queued at refcount 0, so
128
+ // none can be queued for a value still referenced).
129
+ await this.pendingCleanup;
89
130
  await this.lmdbValues.transaction(() => {
90
131
  for (const [hash, val] of values) {
91
132
  this.lmdbValues.put(hash.raw, val.raw);
@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
68
68
  updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
69
69
  getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
70
70
  getState(root: HeaderHash): SerializedState<LeafDb> | null;
71
+ commitFinalized(_headers: HeaderHash[]): void;
71
72
  markUnused(header: HeaderHash): void;
72
73
  diskSizeInBytes(): number | null;
73
74
  close(): Promise<void>;
@@ -1 +1 @@
1
- {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
1
+ {"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
@@ -128,6 +128,11 @@ export class LmdbStates {
128
128
  }
129
129
  return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
130
130
  }
131
+ commitFinalized(_headers) {
132
+ // Values are never pruned here. This db survives restarts, so refcounting
133
+ // would need counts persisted (and crash-consistent) alongside the values.
134
+ // See the TODO above - not implemented until actually needed.
135
+ }
131
136
  markUnused(header) {
132
137
  this.states.removeSync(header.raw);
133
138
  }
@@ -1 +1 @@
1
- {"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBA2F9G"}
1
+ {"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}
@@ -3,9 +3,10 @@ import path from "node:path";
3
3
  import { Block as BlockCodec } from "#@typeberry/block";
4
4
  import { Bytes } from "#@typeberry/bytes";
5
5
  import { Encoder } from "#@typeberry/codec";
6
+ import { RegularStateBackend } from "#@typeberry/config-node";
6
7
  import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
7
8
  import { Logger } from "#@typeberry/logger";
8
- import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
9
+ import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
9
10
  import { getChainSpec, getDatabasePath } from "./common.js";
10
11
  export async function exportBlocks(jamNodeConfig, output, withRelPath) {
11
12
  const logger = Logger.new(import.meta.filename, "export");
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
29
30
  const blake2b = await Blake2b.createHasher();
30
31
  const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
31
32
  const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
32
- const config = LmdbWorkerConfig.new({
33
- nodeName: jamNodeConfig.nodeName,
34
- chainSpec,
35
- blake2b,
36
- dbPath,
37
- workerParams: undefined,
38
- });
39
- const rootDb = config.openDatabase();
33
+ const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
34
+ ? FjallWorkerConfig.new({
35
+ nodeName: jamNodeConfig.nodeName,
36
+ chainSpec,
37
+ blake2b,
38
+ dbPath,
39
+ workerParams: undefined,
40
+ })
41
+ : LmdbWorkerConfig.new({
42
+ nodeName: jamNodeConfig.nodeName,
43
+ chainSpec,
44
+ blake2b,
45
+ dbPath,
46
+ workerParams: undefined,
47
+ });
48
+ const rootDb = await config.openDatabase();
40
49
  const blocks = rootDb.getBlocksDb();
41
50
  logger.info `📖 Gathering blocks...`;
42
51
  const hashes = [];
@@ -1,5 +1,6 @@
1
1
  import { type FuzzVersion } from "#@typeberry/ext-ipc";
2
2
  import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
3
+ import { type Closer } from "#@typeberry/utils";
3
4
  import type { JamConfig } from "./jam-config.js";
4
5
  export type FuzzConfig = {
5
6
  version: FuzzVersion;
@@ -24,5 +25,7 @@ export declare function getFuzzDetails(): {
24
25
  nodeVersion: fuzzV1.Version;
25
26
  gpVersion: fuzzV1.Version;
26
27
  };
27
- export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<() => void>;
28
+ export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<{
29
+ close: Closer;
30
+ }>;
28
31
  //# sourceMappingURL=main-fuzz.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBAuJxF"}
1
+ {"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}