@typeberry/lib 0.9.0-3f2c45b → 0.9.0-4a9f363
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/core/collections/sorted-array.d.ts +1 -1
- package/packages/core/collections/sorted-array.d.ts.map +1 -1
- package/packages/core/collections/sorted-array.js +4 -2
- package/packages/core/collections/sorted-array.test.js +10 -0
- package/packages/core/collections/sorted-set.d.ts +1 -1
- package/packages/core/collections/sorted-set.d.ts.map +1 -1
- package/packages/core/collections/sorted-set.js +6 -2
- package/packages/core/collections/sorted-set.test.js +24 -0
- package/packages/core/utils/index.d.ts +1 -0
- package/packages/core/utils/index.d.ts.map +1 -1
- package/packages/core/utils/index.js +1 -0
- package/packages/core/utils/shutdown.d.ts +18 -0
- package/packages/core/utils/shutdown.d.ts.map +1 -0
- package/packages/core/utils/shutdown.js +3 -0
- package/packages/core/utils/shutdown.node.d.ts +5 -0
- package/packages/core/utils/shutdown.node.d.ts.map +1 -0
- package/packages/core/utils/shutdown.node.js +76 -0
- package/packages/core/utils/shutdown.test.d.ts +2 -0
- package/packages/core/utils/shutdown.test.d.ts.map +1 -0
- package/packages/core/utils/shutdown.test.js +164 -0
- package/packages/extensions/ipc/server.d.ts.map +1 -1
- package/packages/extensions/ipc/server.js +7 -0
- package/packages/jam/config-node/node-config.d.ts +20 -2
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +25 -3
- package/packages/jam/config-node/node-config.test.js +8 -1
- package/packages/jam/database/index.d.ts +1 -0
- package/packages/jam/database/index.d.ts.map +1 -1
- package/packages/jam/database/index.js +1 -0
- package/packages/jam/database/leaf-db-update.d.ts +1 -0
- package/packages/jam/database/leaf-db-update.d.ts.map +1 -1
- package/packages/jam/database/leaf-db-update.js +22 -5
- package/packages/jam/database/leaf-db-update.test.d.ts +2 -0
- package/packages/jam/database/leaf-db-update.test.d.ts.map +1 -0
- package/packages/jam/database/leaf-db-update.test.js +81 -0
- package/packages/jam/database/serialized-states-db.d.ts +1 -0
- package/packages/jam/database/serialized-states-db.d.ts.map +1 -1
- package/packages/jam/database/serialized-states-db.js +4 -0
- package/packages/jam/database/states.d.ts +11 -0
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database/states.js +3 -0
- package/packages/jam/database/value-refs.d.ts +117 -0
- package/packages/jam/database/value-refs.d.ts.map +1 -0
- package/packages/jam/database/value-refs.js +206 -0
- package/packages/jam/database/value-refs.test.d.ts +2 -0
- package/packages/jam/database/value-refs.test.d.ts.map +1 -0
- package/packages/jam/database/value-refs.test.js +165 -0
- package/packages/jam/database-fjall/blocks.d.ts +25 -0
- package/packages/jam/database-fjall/blocks.d.ts.map +1 -0
- package/packages/jam/database-fjall/blocks.js +86 -0
- package/packages/jam/database-fjall/blocks.test.d.ts +2 -0
- package/packages/jam/database-fjall/blocks.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/blocks.test.js +62 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts +14 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.js +55 -6
- package/packages/jam/database-fjall/index.d.ts +2 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -1
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +14 -8
- package/packages/jam/database-fjall/root.d.ts.map +1 -1
- package/packages/jam/database-fjall/root.js +25 -9
- package/packages/jam/database-fjall/states.d.ts +31 -0
- package/packages/jam/database-fjall/states.d.ts.map +1 -0
- package/packages/jam/database-fjall/states.js +99 -0
- package/packages/jam/database-fjall/states.test.d.ts +2 -0
- package/packages/jam/database-fjall/states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/states.test.js +150 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +12 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/hybrid-states.js +45 -4
- package/packages/jam/database-lmdb/states.d.ts +1 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/states.js +5 -0
- package/packages/jam/node/export.d.ts.map +1 -1
- package/packages/jam/node/export.js +18 -9
- package/packages/jam/node/main-fuzz.d.ts +4 -1
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +111 -82
- package/packages/jam/node/main-importer.d.ts +4 -2
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +26 -12
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +42 -15
- package/packages/jam/node/workers.d.ts +4 -4
- package/packages/jam/node/workers.d.ts.map +1 -1
- package/packages/workers/api/config.d.ts +2 -2
- package/packages/workers/api/config.d.ts.map +1 -1
- package/packages/workers/api/config.js +1 -1
- package/packages/workers/api/port.d.ts +5 -1
- package/packages/workers/api/port.d.ts.map +1 -1
- package/packages/workers/api/port.js +56 -13
- package/packages/workers/api/port.test.js +35 -0
- package/packages/workers/api-node/config.d.ts +45 -4
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +127 -9
- package/packages/workers/api-node/config.test.js +52 -3
- package/packages/workers/api-node/port.d.ts +3 -0
- package/packages/workers/api-node/port.d.ts.map +1 -1
- package/packages/workers/api-node/port.js +31 -2
- package/packages/workers/api-node/port.test.js +17 -0
- package/packages/workers/api-node/protocol.d.ts +3 -3
- package/packages/workers/api-node/protocol.d.ts.map +1 -1
- package/packages/workers/api-node/protocol.js +18 -12
- package/packages/workers/block-authorship/main.js +1 -1
- package/packages/workers/importer/main.d.ts +1 -1
- package/packages/workers/importer/main.d.ts.map +1 -1
- package/packages/workers/importer/main.js +1 -1
- package/packages/workers/jam-network/main.js +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { open } from "@fjall-js/fjall";
|
|
3
|
+
import { open, openReadonly, } from "@fjall-js/fjall";
|
|
4
|
+
const DEFAULT_CACHE_SIZE_BYTES = 64 * 1024 * 1024;
|
|
4
5
|
/**
|
|
5
6
|
* Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
|
|
6
7
|
*/
|
|
@@ -29,19 +30,31 @@ export class FjallRoot {
|
|
|
29
30
|
this.options = options;
|
|
30
31
|
}
|
|
31
32
|
/** Open (or create) a fjall keyspace at the given path. */
|
|
32
|
-
static async open(dbPath, options) {
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
cacheSizeBytes: options.cacheSizeBytes,
|
|
38
|
-
}
|
|
33
|
+
static async open(dbPath, options = {}) {
|
|
34
|
+
// fjall-js 0.3 shares one engine per path. Readers must use the read-only
|
|
35
|
+
// wrapper surface; durability is driven explicitly through persist().
|
|
36
|
+
const config = {
|
|
37
|
+
path: dbPath,
|
|
38
|
+
cacheSizeBytes: options.cacheSizeBytes ?? DEFAULT_CACHE_SIZE_BYTES,
|
|
39
|
+
};
|
|
40
|
+
const keyspace = options.readOnly === true ? await openReadonly(config) : await open(config);
|
|
39
41
|
return new FjallRoot(keyspace, dbPath, options);
|
|
40
42
|
}
|
|
43
|
+
/** Whether this root was opened through fjall's read-only surface. */
|
|
44
|
+
get readOnly() {
|
|
45
|
+
return this.options.readOnly === true;
|
|
46
|
+
}
|
|
41
47
|
/** Open (or create) a partition under this keyspace. */
|
|
42
48
|
async partition(name) {
|
|
43
49
|
return this.keyspace.partition(name);
|
|
44
50
|
}
|
|
51
|
+
/** Open a writable partition, failing early when this root is read-only. */
|
|
52
|
+
async writablePartition(name) {
|
|
53
|
+
if (this.readOnly) {
|
|
54
|
+
throw new Error(`Cannot open writable fjall partition '${name}' from a read-only keyspace.`);
|
|
55
|
+
}
|
|
56
|
+
return (await this.keyspace.partition(name));
|
|
57
|
+
}
|
|
45
58
|
/**
|
|
46
59
|
* Flush the journal to disk so prior writes survive a crash.
|
|
47
60
|
*
|
|
@@ -52,6 +65,9 @@ export class FjallRoot {
|
|
|
52
65
|
if (this.options.ephemeral === true) {
|
|
53
66
|
return;
|
|
54
67
|
}
|
|
68
|
+
if (this.readOnly) {
|
|
69
|
+
throw new Error("Cannot persist a read-only fjall keyspace.");
|
|
70
|
+
}
|
|
55
71
|
await this.keyspace.persist();
|
|
56
72
|
}
|
|
57
73
|
/**
|
|
@@ -69,7 +85,7 @@ export class FjallRoot {
|
|
|
69
85
|
return null;
|
|
70
86
|
}
|
|
71
87
|
}
|
|
72
|
-
/**
|
|
88
|
+
/** Release this keyspace handle. Call persist() first when durability is needed. */
|
|
73
89
|
async close() {
|
|
74
90
|
await this.keyspace.close();
|
|
75
91
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { HeaderHash, StateRootHash } from "#@typeberry/block";
|
|
2
|
+
import type { ChainSpec } from "#@typeberry/config";
|
|
3
|
+
import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
|
|
4
|
+
import type { Blake2b } from "#@typeberry/hash";
|
|
5
|
+
import type { ServicesUpdate, State } from "#@typeberry/state";
|
|
6
|
+
import type { StateEntries } from "#@typeberry/state-merkleization";
|
|
7
|
+
import { SerializedState } from "#@typeberry/state-merkleization";
|
|
8
|
+
import { OK, Result } from "#@typeberry/utils";
|
|
9
|
+
import { type FjallRoot } from "./root.js";
|
|
10
|
+
/** fjall-backed full on-disk serialized state storage. */
|
|
11
|
+
export declare class FjallStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
|
|
12
|
+
private readonly spec;
|
|
13
|
+
private readonly blake2b;
|
|
14
|
+
private readonly root;
|
|
15
|
+
private readonly states;
|
|
16
|
+
private readonly values;
|
|
17
|
+
static open(spec: ChainSpec, blake2b: Blake2b, root: FjallRoot): Promise<FjallStates>;
|
|
18
|
+
private readonly valuesDb;
|
|
19
|
+
private constructor();
|
|
20
|
+
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
21
|
+
updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
22
|
+
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
23
|
+
getState(headerHash: HeaderHash): SerializedState<LeafDb> | null;
|
|
24
|
+
commitFinalized(_headers: HeaderHash[]): void;
|
|
25
|
+
markUnused(headerHash: HeaderHash): void;
|
|
26
|
+
diskSizeInBytes(): number | null;
|
|
27
|
+
close(): Promise<void>;
|
|
28
|
+
private updateAndCommit;
|
|
29
|
+
private readValue;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=states.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAE/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAI9F,0DAA0D;AAC1D,qBAAa,WAAY,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS7F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;WAZZ,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAK3F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,OAAO;IAUD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAQxG,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAUlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAahE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAK7C,UAAU,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMxC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;YAEG,eAAe;IAyB7B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { BytesBlob } from "#@typeberry/bytes";
|
|
2
|
+
import { SortedSet } from "#@typeberry/collections";
|
|
3
|
+
import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
|
|
4
|
+
import { Logger } from "#@typeberry/logger";
|
|
5
|
+
import { SerializedState, StateEntryUpdateAction, serializeStateUpdate } from "#@typeberry/state-merkleization";
|
|
6
|
+
import { leafComparator } from "#@typeberry/trie";
|
|
7
|
+
import { OK, Result, resultToString } from "#@typeberry/utils";
|
|
8
|
+
import { toUint8Array } from "./root.js";
|
|
9
|
+
const logger = Logger.new(import.meta.filename, "db");
|
|
10
|
+
/** fjall-backed full on-disk serialized state storage. */
|
|
11
|
+
export class FjallStates {
|
|
12
|
+
spec;
|
|
13
|
+
blake2b;
|
|
14
|
+
root;
|
|
15
|
+
states;
|
|
16
|
+
values;
|
|
17
|
+
static async open(spec, blake2b, root) {
|
|
18
|
+
const [states, values] = await Promise.all([root.partition("states"), root.partition("values")]);
|
|
19
|
+
return new FjallStates(spec, blake2b, root, states, values);
|
|
20
|
+
}
|
|
21
|
+
valuesDb;
|
|
22
|
+
constructor(spec, blake2b, root, states, values) {
|
|
23
|
+
this.spec = spec;
|
|
24
|
+
this.blake2b = blake2b;
|
|
25
|
+
this.root = root;
|
|
26
|
+
this.states = states;
|
|
27
|
+
this.values = values;
|
|
28
|
+
this.valuesDb = { get: (key) => this.readValue(key) };
|
|
29
|
+
}
|
|
30
|
+
async insertInitialState(headerHash, entries) {
|
|
31
|
+
return await this.updateAndCommit(headerHash, SortedSet.fromArray(leafComparator, []), Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
|
|
32
|
+
}
|
|
33
|
+
async updateAndSetState(headerHash, state, update) {
|
|
34
|
+
const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
|
|
35
|
+
const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
|
|
36
|
+
const res = await this.updateAndCommit(headerHash, newLeafs, updatedValues);
|
|
37
|
+
if (res.isOk) {
|
|
38
|
+
state.updateBackend(LeafDb.fromLeaves(newLeafs, this.valuesDb));
|
|
39
|
+
}
|
|
40
|
+
return res;
|
|
41
|
+
}
|
|
42
|
+
async getStateRoot(state) {
|
|
43
|
+
return state.backend.getStateRoot(this.blake2b);
|
|
44
|
+
}
|
|
45
|
+
getState(headerHash) {
|
|
46
|
+
const leafNodes = toUint8Array(this.states.get(headerHash.raw));
|
|
47
|
+
if (leafNodes === null) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
const leafDbResult = LeafDb.fromLeavesBlob(BytesBlob.blobFrom(leafNodes), this.valuesDb);
|
|
51
|
+
if (leafDbResult.isError) {
|
|
52
|
+
throw new Error(`Inconsistent DB. Invalid leaf nodes for ${headerHash}: ${resultToString(leafDbResult)}`);
|
|
53
|
+
}
|
|
54
|
+
return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
|
|
55
|
+
}
|
|
56
|
+
commitFinalized(_headers) {
|
|
57
|
+
// Values are never pruned here. This db survives restarts, so refcounting
|
|
58
|
+
// would need counts persisted (and crash-consistent) alongside the values.
|
|
59
|
+
}
|
|
60
|
+
markUnused(headerHash) {
|
|
61
|
+
void writable(this.states, this.root)
|
|
62
|
+
.remove(headerHash.raw)
|
|
63
|
+
.catch((e) => logger.warn `Failed to prune state ${headerHash}: ${e}`);
|
|
64
|
+
}
|
|
65
|
+
diskSizeInBytes() {
|
|
66
|
+
return this.root.sizeInBytes();
|
|
67
|
+
}
|
|
68
|
+
async close() { }
|
|
69
|
+
async updateAndCommit(headerHash, leafs, data) {
|
|
70
|
+
const { values } = updateLeafs(leafs, this.blake2b, data);
|
|
71
|
+
const stateLeafs = BytesBlob.blobFromParts(leafs.array.map((x) => x.node.raw));
|
|
72
|
+
try {
|
|
73
|
+
// Preserve dependency order: values first, then leaves that may reference them.
|
|
74
|
+
if (values.length > 0) {
|
|
75
|
+
await writable(this.values, this.root).insertBatch(values.map(([hash, val]) => ({ key: hash.raw, value: val.raw })));
|
|
76
|
+
}
|
|
77
|
+
await writable(this.states, this.root).insert(headerHash.raw, stateLeafs.raw);
|
|
78
|
+
await this.root.persist();
|
|
79
|
+
}
|
|
80
|
+
catch (e) {
|
|
81
|
+
logger.error `${e}`;
|
|
82
|
+
return Result.error(StateUpdateError.Commit, () => `Failed to commit state update: ${e}`);
|
|
83
|
+
}
|
|
84
|
+
return Result.ok(OK);
|
|
85
|
+
}
|
|
86
|
+
readValue(key) {
|
|
87
|
+
const val = toUint8Array(this.values.get(key.raw));
|
|
88
|
+
if (val === null) {
|
|
89
|
+
throw new Error(`Missing required value: ${key} in the DB`);
|
|
90
|
+
}
|
|
91
|
+
return val;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function writable(partition, root) {
|
|
95
|
+
if (root.readOnly) {
|
|
96
|
+
throw new Error("Cannot write through a read-only fjall partition.");
|
|
97
|
+
}
|
|
98
|
+
return partition;
|
|
99
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { afterEach, before, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import { tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "#@typeberry/block";
|
|
5
|
+
import { Bytes, BytesBlob } from "#@typeberry/bytes";
|
|
6
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
7
|
+
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
8
|
+
import { tryAsU32, tryAsU64 } from "#@typeberry/numbers";
|
|
9
|
+
import { InMemoryState, LookupHistoryItem, PrivilegedServices, ServiceAccountInfo, tryAsLookupHistorySlots, tryAsPerCore, UpdateService, } from "#@typeberry/state";
|
|
10
|
+
import { testState } from "#@typeberry/state/test.utils.js";
|
|
11
|
+
import { StateEntries } from "#@typeberry/state-merkleization";
|
|
12
|
+
import { deepEqual, OK, Result } from "#@typeberry/utils";
|
|
13
|
+
import { FjallRoot } from "./root.js";
|
|
14
|
+
import { FjallStates } from "./states.js";
|
|
15
|
+
let blake2b;
|
|
16
|
+
before(async () => {
|
|
17
|
+
blake2b = await Blake2b.createHasher();
|
|
18
|
+
});
|
|
19
|
+
describe("Fjall states database", () => {
|
|
20
|
+
const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
21
|
+
const spec = tinyChainSpec;
|
|
22
|
+
let tmpDir = "";
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
tmpDir = fs.mkdtempSync("typeberry-fjall-states-");
|
|
25
|
+
});
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
28
|
+
});
|
|
29
|
+
it("imports and reads an empty state", async () => {
|
|
30
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
31
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
32
|
+
try {
|
|
33
|
+
const emptyState = InMemoryState.empty(spec);
|
|
34
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, emptyState);
|
|
35
|
+
const emptyRoot = serialized.getRootHash(blake2b);
|
|
36
|
+
deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
|
|
37
|
+
const newState = states.getState(headerHash);
|
|
38
|
+
assert.ok(newState !== null);
|
|
39
|
+
assert.strictEqual(`${await states.getStateRoot(newState)}`, `${emptyRoot}`);
|
|
40
|
+
deepEqual(InMemoryState.copyFrom(spec, newState, new Map()), emptyState);
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await states.close();
|
|
44
|
+
await root.close();
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
it("reads large values back from disk", async () => {
|
|
48
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
49
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
50
|
+
try {
|
|
51
|
+
const big1 = BytesBlob.blobFromString("x".repeat(100));
|
|
52
|
+
const big2 = BytesBlob.blobFromString("y".repeat(100));
|
|
53
|
+
const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
|
|
54
|
+
const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
55
|
+
const entries = StateEntries.fromEntriesUnsafe([
|
|
56
|
+
[key1, big1],
|
|
57
|
+
[key2, big2],
|
|
58
|
+
]);
|
|
59
|
+
deepEqual(await states.insertInitialState(headerHash, entries), Result.ok(OK));
|
|
60
|
+
const state = states.getState(headerHash);
|
|
61
|
+
assert.ok(state !== null);
|
|
62
|
+
assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
|
|
63
|
+
assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
await states.close();
|
|
67
|
+
await root.close();
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
it("updates state", async () => {
|
|
71
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
72
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
73
|
+
try {
|
|
74
|
+
const state = InMemoryState.empty(spec);
|
|
75
|
+
await states.insertInitialState(headerHash, StateEntries.serializeInMemory(spec, blake2b, state));
|
|
76
|
+
const newState = states.getState(headerHash);
|
|
77
|
+
assert.ok(newState !== null);
|
|
78
|
+
const headerHash2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
79
|
+
const lookupHistory = LookupHistoryItem.new(Bytes.fill(HASH_SIZE, 0xff).asOpaque(), tryAsU32(5), tryAsLookupHistorySlots([]));
|
|
80
|
+
const stateUpdate = {
|
|
81
|
+
timeslot: tryAsTimeSlot(15),
|
|
82
|
+
privilegedServices: PrivilegedServices.create({
|
|
83
|
+
manager: tryAsServiceId(1),
|
|
84
|
+
assigners: tryAsPerCore(new Array(spec.coresCount).fill(tryAsServiceId(2)), spec),
|
|
85
|
+
delegator: tryAsServiceId(3),
|
|
86
|
+
registrar: tryAsServiceId(4),
|
|
87
|
+
autoAccumulateServices: new Map(),
|
|
88
|
+
}),
|
|
89
|
+
updated: new Map([
|
|
90
|
+
[
|
|
91
|
+
tryAsServiceId(1),
|
|
92
|
+
UpdateService.create({
|
|
93
|
+
serviceInfo: ServiceAccountInfo.create({
|
|
94
|
+
codeHash: Bytes.zero(HASH_SIZE).asOpaque(),
|
|
95
|
+
balance: tryAsU64(1_000_000),
|
|
96
|
+
accumulateMinGas: tryAsServiceGas(10_000),
|
|
97
|
+
onTransferMinGas: tryAsServiceGas(5_000),
|
|
98
|
+
storageUtilisationBytes: tryAsU64(1_000),
|
|
99
|
+
gratisStorage: tryAsU64(0),
|
|
100
|
+
storageUtilisationCount: tryAsU32(1),
|
|
101
|
+
created: tryAsTimeSlot(0),
|
|
102
|
+
lastAccumulation: tryAsTimeSlot(0),
|
|
103
|
+
parentService: tryAsServiceId(0),
|
|
104
|
+
}),
|
|
105
|
+
lookupHistory,
|
|
106
|
+
}),
|
|
107
|
+
],
|
|
108
|
+
]),
|
|
109
|
+
};
|
|
110
|
+
deepEqual(state.applyUpdate(stateUpdate), Result.ok(OK));
|
|
111
|
+
deepEqual(await states.updateAndSetState(headerHash2, newState, stateUpdate), Result.ok(OK));
|
|
112
|
+
const updatedState = states.getState(headerHash2);
|
|
113
|
+
assert.ok(updatedState !== null);
|
|
114
|
+
deepEqual(InMemoryState.copyFrom(spec, updatedState, new Map([
|
|
115
|
+
[
|
|
116
|
+
tryAsServiceId(1),
|
|
117
|
+
{
|
|
118
|
+
storageKeys: [],
|
|
119
|
+
preimages: [],
|
|
120
|
+
lookupHistory: [{ hash: lookupHistory.hash, length: lookupHistory.length }],
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
])), state);
|
|
124
|
+
assert.strictEqual(`${await states.getStateRoot(updatedState)}`, `${StateEntries.serializeInMemory(spec, blake2b, state).getRootHash(blake2b)}`);
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
await states.close();
|
|
128
|
+
await root.close();
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
it("imports a more complex state", async () => {
|
|
132
|
+
const root = await FjallRoot.open(tmpDir, { ephemeral: true });
|
|
133
|
+
const states = await FjallStates.open(spec, blake2b, root);
|
|
134
|
+
try {
|
|
135
|
+
const initialState = testState();
|
|
136
|
+
const initialService = initialState.services.get(tryAsServiceId(0));
|
|
137
|
+
assert.ok(initialService !== undefined);
|
|
138
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, initialState);
|
|
139
|
+
deepEqual(await states.insertInitialState(headerHash, serialized), Result.ok(OK));
|
|
140
|
+
const newState = states.getState(headerHash);
|
|
141
|
+
assert.ok(newState !== null);
|
|
142
|
+
assert.strictEqual(`${await states.getStateRoot(newState)}`, `${serialized.getRootHash(blake2b)}`);
|
|
143
|
+
deepEqual(InMemoryState.copyFrom(spec, newState, new Map([[initialService.serviceId, initialService.getEntries()]])), initialState);
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
await states.close();
|
|
147
|
+
await root.close();
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -12,6 +12,12 @@ import { OK, Result } from "#@typeberry/utils";
|
|
|
12
12
|
* Reads go straight to lmdb, which keeps its own page cache.
|
|
13
13
|
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
14
14
|
* keep the heap usage bounded.
|
|
15
|
+
*
|
|
16
|
+
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
17
|
+
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
18
|
+
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
19
|
+
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
20
|
+
* run are never collected.
|
|
15
21
|
*/
|
|
16
22
|
export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
|
|
17
23
|
private readonly spec;
|
|
@@ -20,6 +26,9 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
20
26
|
private readonly inMemStates;
|
|
21
27
|
private readonly lmdbValues;
|
|
22
28
|
private readonly valuesDb;
|
|
29
|
+
private readonly refsStore;
|
|
30
|
+
private readonly refs;
|
|
31
|
+
private pendingCleanup;
|
|
23
32
|
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
|
|
24
33
|
spec: ChainSpec;
|
|
25
34
|
blake2b: Blake2b;
|
|
@@ -33,7 +42,10 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
33
42
|
updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
34
43
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
35
44
|
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
45
|
+
commitFinalized(headers: HeaderHash[]): void;
|
|
36
46
|
markUnused(header: HeaderHash): void;
|
|
47
|
+
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
48
|
+
private applyRefs;
|
|
37
49
|
diskSizeInBytes(): number | null;
|
|
38
50
|
close(): Promise<void>;
|
|
39
51
|
/** Write new large values to LMDB in one transaction. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,
|
|
1
|
+
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { HashDictionary, SortedSet } from "#@typeberry/collections";
|
|
2
|
-
import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
|
|
2
|
+
import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
|
|
3
|
+
import { Logger } from "#@typeberry/logger";
|
|
3
4
|
import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
|
|
4
5
|
import { leafComparator } from "#@typeberry/trie";
|
|
5
6
|
import { OK, Result } from "#@typeberry/utils";
|
|
6
7
|
import { LmdbRoot } from "./root.js";
|
|
8
|
+
const logger = Logger.new(import.meta.filename, "db");
|
|
7
9
|
/**
|
|
8
10
|
* Hybrid serialized-states db.
|
|
9
11
|
*
|
|
@@ -11,6 +13,12 @@ import { LmdbRoot } from "./root.js";
|
|
|
11
13
|
* Reads go straight to lmdb, which keeps its own page cache.
|
|
12
14
|
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
13
15
|
* keep the heap usage bounded.
|
|
16
|
+
*
|
|
17
|
+
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
18
|
+
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
19
|
+
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
20
|
+
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
21
|
+
* run are never collected.
|
|
14
22
|
*/
|
|
15
23
|
export class HybridSerializedStates {
|
|
16
24
|
spec;
|
|
@@ -20,6 +28,10 @@ export class HybridSerializedStates {
|
|
|
20
28
|
lmdbValues;
|
|
21
29
|
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
22
30
|
valuesDb;
|
|
31
|
+
refsStore = new InMemoryValueRefsStore();
|
|
32
|
+
refs = new ValueRefs(this.refsStore);
|
|
33
|
+
// Queue of not-yet-committed value removals, awaited on close.
|
|
34
|
+
pendingCleanup = Promise.resolve();
|
|
23
35
|
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
|
|
24
36
|
const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
|
|
25
37
|
return new HybridSerializedStates(spec, blake2b, root);
|
|
@@ -38,13 +50,14 @@ export class HybridSerializedStates {
|
|
|
38
50
|
return res;
|
|
39
51
|
}
|
|
40
52
|
this.inMemStates.set(headerHash, leafs);
|
|
53
|
+
this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
|
|
41
54
|
return Result.ok(OK);
|
|
42
55
|
}
|
|
43
56
|
async updateAndSetState(header, state, update) {
|
|
44
57
|
const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
|
|
45
58
|
// Clone the leaf set before mutating: the previous state keeps using its own.
|
|
46
59
|
const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
|
|
47
|
-
const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
60
|
+
const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
48
61
|
const res = await this.writeValues(values);
|
|
49
62
|
if (res.isError) {
|
|
50
63
|
// Leave the caller's state untouched: its new leaves would reference
|
|
@@ -55,6 +68,7 @@ export class HybridSerializedStates {
|
|
|
55
68
|
// values are durably written.
|
|
56
69
|
state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
|
|
57
70
|
this.inMemStates.set(header, leafs);
|
|
71
|
+
this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
|
|
58
72
|
return Result.ok(OK);
|
|
59
73
|
}
|
|
60
74
|
async getStateRoot(state) {
|
|
@@ -68,15 +82,37 @@ export class HybridSerializedStates {
|
|
|
68
82
|
const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
|
|
69
83
|
return SerializedState.new(this.spec, this.blake2b, leafDb);
|
|
70
84
|
}
|
|
85
|
+
commitFinalized(headers) {
|
|
86
|
+
this.applyRefs(this.refs.commitFinalized(headers));
|
|
87
|
+
}
|
|
71
88
|
markUnused(header) {
|
|
72
|
-
//
|
|
73
|
-
//
|
|
89
|
+
// Release the speculative references first (a no-op for finalized states,
|
|
90
|
+
// whose deltas were already consumed by `commitFinalized`).
|
|
91
|
+
this.applyRefs(this.refs.releaseUnfinalized(header));
|
|
74
92
|
this.inMemStates.delete(header);
|
|
75
93
|
}
|
|
94
|
+
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
95
|
+
applyRefs(update) {
|
|
96
|
+
this.refsStore.apply(update);
|
|
97
|
+
if (update.removeValues.length === 0) {
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
// Queued, not awaited: a failed removal only leaks a value.
|
|
101
|
+
this.pendingCleanup = this.pendingCleanup
|
|
102
|
+
.then(() => this.lmdbValues.transaction(() => {
|
|
103
|
+
for (const v of update.removeValues) {
|
|
104
|
+
this.lmdbValues.remove(v.raw);
|
|
105
|
+
}
|
|
106
|
+
}))
|
|
107
|
+
.catch((e) => {
|
|
108
|
+
logger.error `Failed to remove unreferenced values: ${e}`;
|
|
109
|
+
});
|
|
110
|
+
}
|
|
76
111
|
diskSizeInBytes() {
|
|
77
112
|
return this.root.sizeInBytes();
|
|
78
113
|
}
|
|
79
114
|
async close() {
|
|
115
|
+
await this.pendingCleanup;
|
|
80
116
|
await this.lmdbValues.close();
|
|
81
117
|
await this.root.close();
|
|
82
118
|
}
|
|
@@ -86,6 +122,11 @@ export class HybridSerializedStates {
|
|
|
86
122
|
return Result.ok(OK);
|
|
87
123
|
}
|
|
88
124
|
try {
|
|
125
|
+
// Flush queued removals first: a pending cleanup might be about to delete
|
|
126
|
+
// a content hash we are re-inserting now. Writing behind it keeps the
|
|
127
|
+
// re-inserted value on disk (removals are only queued at refcount 0, so
|
|
128
|
+
// none can be queued for a value still referenced).
|
|
129
|
+
await this.pendingCleanup;
|
|
89
130
|
await this.lmdbValues.transaction(() => {
|
|
90
131
|
for (const [hash, val] of values) {
|
|
91
132
|
this.lmdbValues.put(hash.raw, val.raw);
|
|
@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
|
|
|
68
68
|
updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
69
69
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
70
70
|
getState(root: HeaderHash): SerializedState<LeafDb> | null;
|
|
71
|
+
commitFinalized(_headers: HeaderHash[]): void;
|
|
71
72
|
markUnused(header: HeaderHash): void;
|
|
72
73
|
diskSizeInBytes(): number | null;
|
|
73
74
|
close(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
|
|
@@ -128,6 +128,11 @@ export class LmdbStates {
|
|
|
128
128
|
}
|
|
129
129
|
return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
|
|
130
130
|
}
|
|
131
|
+
commitFinalized(_headers) {
|
|
132
|
+
// Values are never pruned here. This db survives restarts, so refcounting
|
|
133
|
+
// would need counts persisted (and crash-consistent) alongside the values.
|
|
134
|
+
// See the TODO above - not implemented until actually needed.
|
|
135
|
+
}
|
|
131
136
|
markUnused(header) {
|
|
132
137
|
this.states.removeSync(header.raw);
|
|
133
138
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}
|
|
@@ -3,9 +3,10 @@ import path from "node:path";
|
|
|
3
3
|
import { Block as BlockCodec } from "#@typeberry/block";
|
|
4
4
|
import { Bytes } from "#@typeberry/bytes";
|
|
5
5
|
import { Encoder } from "#@typeberry/codec";
|
|
6
|
+
import { RegularStateBackend } from "#@typeberry/config-node";
|
|
6
7
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
7
8
|
import { Logger } from "#@typeberry/logger";
|
|
8
|
-
import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
|
|
9
|
+
import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
|
|
9
10
|
import { getChainSpec, getDatabasePath } from "./common.js";
|
|
10
11
|
export async function exportBlocks(jamNodeConfig, output, withRelPath) {
|
|
11
12
|
const logger = Logger.new(import.meta.filename, "export");
|
|
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
|
|
|
29
30
|
const blake2b = await Blake2b.createHasher();
|
|
30
31
|
const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
|
|
31
32
|
const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
|
|
32
|
-
const config =
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
|
|
34
|
+
? FjallWorkerConfig.new({
|
|
35
|
+
nodeName: jamNodeConfig.nodeName,
|
|
36
|
+
chainSpec,
|
|
37
|
+
blake2b,
|
|
38
|
+
dbPath,
|
|
39
|
+
workerParams: undefined,
|
|
40
|
+
})
|
|
41
|
+
: LmdbWorkerConfig.new({
|
|
42
|
+
nodeName: jamNodeConfig.nodeName,
|
|
43
|
+
chainSpec,
|
|
44
|
+
blake2b,
|
|
45
|
+
dbPath,
|
|
46
|
+
workerParams: undefined,
|
|
47
|
+
});
|
|
48
|
+
const rootDb = await config.openDatabase();
|
|
40
49
|
const blocks = rootDb.getBlocksDb();
|
|
41
50
|
logger.info `📖 Gathering blocks...`;
|
|
42
51
|
const hashes = [];
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type FuzzVersion } from "#@typeberry/ext-ipc";
|
|
2
2
|
import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
|
|
3
|
+
import { type Closer } from "#@typeberry/utils";
|
|
3
4
|
import type { JamConfig } from "./jam-config.js";
|
|
4
5
|
export type FuzzConfig = {
|
|
5
6
|
version: FuzzVersion;
|
|
@@ -24,5 +25,7 @@ export declare function getFuzzDetails(): {
|
|
|
24
25
|
nodeVersion: fuzzV1.Version;
|
|
25
26
|
gpVersion: fuzzV1.Version;
|
|
26
27
|
};
|
|
27
|
-
export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<
|
|
28
|
+
export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<{
|
|
29
|
+
close: Closer;
|
|
30
|
+
}>;
|
|
28
31
|
//# sourceMappingURL=main-fuzz.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}
|