@typeberry/lib 0.11.0-829b7ea → 0.11.0-9d838d3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/jam/database-fjall/blocks.d.ts +1 -0
- package/packages/jam/database-fjall/blocks.d.ts.map +1 -1
- package/packages/jam/database-fjall/blocks.js +8 -3
- package/packages/jam/database-fjall/hybrid-states.d.ts +10 -4
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.js +22 -6
- package/packages/jam/database-fjall/root.d.ts +2 -0
- package/packages/jam/database-fjall/root.d.ts.map +1 -1
- package/packages/jam/database-fjall/root.js +8 -1
- package/packages/jam/database-fjall/root.test.d.ts +2 -0
- package/packages/jam/database-fjall/root.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/root.test.js +23 -0
- package/packages/jam/database-fjall/states.d.ts +1 -0
- package/packages/jam/database-fjall/states.d.ts.map +1 -1
- package/packages/jam/database-fjall/states.js +6 -3
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +62 -42
- package/packages/jam/node/main-importer.d.ts +4 -5
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +3 -2
- package/packages/workers/api-node/config.d.ts +10 -8
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +28 -17
- package/packages/workers/api-node/config.test.js +56 -0
package/package.json
CHANGED
|
@@ -12,6 +12,7 @@ export declare class FjallBlocks implements BlocksDb {
|
|
|
12
12
|
private readonly postStateRoots;
|
|
13
13
|
static open(chainSpec: ChainSpec, root: FjallRoot): Promise<FjallBlocks>;
|
|
14
14
|
private constructor();
|
|
15
|
+
private pendingPrune;
|
|
15
16
|
setPostStateRoot(hash: HeaderHash, postStateRoot: StateRootHash): Promise<void>;
|
|
16
17
|
getPostStateRoot(hash: HeaderHash): StateRootHash | null;
|
|
17
18
|
insertBlock(block: WithHash<HeaderHash, BlockView>): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAK9F,kCAAkC;AAClC,qBAAa,WAAY,YAAW,QAAQ;IAWxC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc;WAdpB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAS9E,OAAO;
|
|
1
|
+
{"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAK9F,kCAAkC;AAClC,qBAAa,WAAY,YAAW,QAAQ;IAWxC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc;WAdpB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAS9E,OAAO;IAQP,OAAO,CAAC,YAAY,CAAuC;IAErD,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrF,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQlD,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlE,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxD,iBAAiB,IAAI,UAAU;IAQ/B,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAQ9C,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQpD,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAY5B,KAAK;CAGZ"}
|
|
@@ -28,6 +28,7 @@ export class FjallBlocks {
|
|
|
28
28
|
this.extrinsics = extrinsics;
|
|
29
29
|
this.postStateRoots = postStateRoots;
|
|
30
30
|
}
|
|
31
|
+
pendingPrune = Promise.resolve();
|
|
31
32
|
async setPostStateRoot(hash, postStateRoot) {
|
|
32
33
|
await writable(this.postStateRoots, this.root).insert(hash.raw, postStateRoot.raw);
|
|
33
34
|
}
|
|
@@ -70,13 +71,17 @@ export class FjallBlocks {
|
|
|
70
71
|
return Decoder.decodeObject(Extrinsic.Codec.View, data, this.chainSpec);
|
|
71
72
|
}
|
|
72
73
|
markUnused(hash) {
|
|
73
|
-
|
|
74
|
+
this.pendingPrune = this.pendingPrune
|
|
75
|
+
.then(() => Promise.all([
|
|
74
76
|
writable(this.headers, this.root).remove(hash.raw),
|
|
75
77
|
writable(this.extrinsics, this.root).remove(hash.raw),
|
|
76
78
|
writable(this.postStateRoots, this.root).remove(hash.raw),
|
|
77
|
-
])
|
|
79
|
+
]))
|
|
80
|
+
.catch((e) => logger.warn `Failed to prune block ${hash}: ${e}`);
|
|
81
|
+
}
|
|
82
|
+
async close() {
|
|
83
|
+
await this.pendingPrune;
|
|
78
84
|
}
|
|
79
|
-
async close() { }
|
|
80
85
|
}
|
|
81
86
|
function writable(partition, root) {
|
|
82
87
|
if (root.readOnly) {
|
|
@@ -5,13 +5,13 @@ import type { Blake2b } from "#@typeberry/hash";
|
|
|
5
5
|
import type { ServicesUpdate, State } from "#@typeberry/state";
|
|
6
6
|
import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
|
|
7
7
|
import { OK, Result } from "#@typeberry/utils";
|
|
8
|
-
import { type FjallRootOptions, type Partition } from "./root.js";
|
|
8
|
+
import { FjallRoot, type FjallRootOptions, type Partition } from "./root.js";
|
|
9
9
|
/**
|
|
10
10
|
* One open fjall keyspace together with its content-addressed `values`
|
|
11
11
|
* partition.
|
|
12
12
|
*
|
|
13
|
-
* Opening the keyspace is the slow part, so the fuzz target
|
|
14
|
-
* per run and
|
|
13
|
+
* Opening the keyspace is the slow part, so the fuzz target can open one root
|
|
14
|
+
* per run and reuse it for every reset (see `HybridSerializedStates.fromRoot`).
|
|
15
15
|
* The values partition is immutable - the key is the hash of the value - so it
|
|
16
16
|
* is fine that values pile up across resets, the unreferenced ones just sit
|
|
17
17
|
* there unused.
|
|
@@ -20,9 +20,13 @@ export declare class FjallValuesSession {
|
|
|
20
20
|
private readonly root;
|
|
21
21
|
/** Shared content-addressed values partition, reused across resets. */
|
|
22
22
|
readonly values: Partition;
|
|
23
|
+
/** Whether close() should release the underlying keyspace. */
|
|
24
|
+
private readonly ownsRoot;
|
|
23
25
|
private constructor();
|
|
24
26
|
/** Open (or create) the keyspace at `dbPath` and its `values` partition. */
|
|
25
27
|
static open(dbPath: string, options?: FjallRootOptions): Promise<FjallValuesSession>;
|
|
28
|
+
/** Wrap an already-open keyspace and open its `values` partition. */
|
|
29
|
+
static fromRoot(root: FjallRoot): Promise<FjallValuesSession>;
|
|
26
30
|
/** Flush the journal to disk (a no-op for ephemeral keyspaces). */
|
|
27
31
|
persist(): Promise<void>;
|
|
28
32
|
/** Size of the keyspace directory on disk, in bytes. */
|
|
@@ -44,7 +48,7 @@ export declare class FjallValuesSession {
|
|
|
44
48
|
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
45
49
|
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
46
50
|
* anyway (the leaf sets live in memory), so values left over by a previous run
|
|
47
|
-
* are never collected. An instance backed by a shared
|
|
51
|
+
* are never collected. An instance backed by a shared keyspace (fuzz reset
|
|
48
52
|
* reuse) only ever prunes values it inserted itself, since its refcounts start
|
|
49
53
|
* empty - values left behind by earlier resets stay untouched.
|
|
50
54
|
*/
|
|
@@ -61,6 +65,8 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
61
65
|
ephemeral?: boolean;
|
|
62
66
|
cacheSizeBytes?: number;
|
|
63
67
|
}): Promise<HybridSerializedStates>;
|
|
68
|
+
/** Create a db over an already-open keyspace owned by the caller. */
|
|
69
|
+
static fromRoot(spec: ChainSpec, blake2b: Blake2b, root: FjallRoot): Promise<HybridSerializedStates>;
|
|
64
70
|
/**
|
|
65
71
|
* Wrap an already-open `FjallValuesSession` and reuse its keyspace.
|
|
66
72
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,
|
|
1
|
+
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAgB,MAAM,WAAW,CAAC;AAI3F;;;;;;;;;GASG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,SAAS;IAC1B,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAL3B,OAAO;IAQP,4EAA4E;WAC/D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAc9F,qEAAqE;WACxD,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAKnE,mEAAmE;IAC7D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,wDAAwD;IACxD,WAAW,IAAI,MAAM,GAAG,IAAI;IAI5B,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAgDxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,WAAW;WAnDjB,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,cAAc,GACf,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC,qEAAqE;WACxD,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAK1G;;;;;;;OAOG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,GAAG,sBAAsB;IAI1G,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,OAAO;IAWD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAIhC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAcX,KAAK;IAWX,qEAAqE;YACvD,WAAW;IAoBzB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -10,8 +10,8 @@ const logger = Logger.new(import.meta.filename, "db");
|
|
|
10
10
|
* One open fjall keyspace together with its content-addressed `values`
|
|
11
11
|
* partition.
|
|
12
12
|
*
|
|
13
|
-
* Opening the keyspace is the slow part, so the fuzz target
|
|
14
|
-
* per run and
|
|
13
|
+
* Opening the keyspace is the slow part, so the fuzz target can open one root
|
|
14
|
+
* per run and reuse it for every reset (see `HybridSerializedStates.fromRoot`).
|
|
15
15
|
* The values partition is immutable - the key is the hash of the value - so it
|
|
16
16
|
* is fine that values pile up across resets, the unreferenced ones just sit
|
|
17
17
|
* there unused.
|
|
@@ -19,11 +19,15 @@ const logger = Logger.new(import.meta.filename, "db");
|
|
|
19
19
|
export class FjallValuesSession {
|
|
20
20
|
root;
|
|
21
21
|
values;
|
|
22
|
+
ownsRoot;
|
|
22
23
|
constructor(root,
|
|
23
24
|
/** Shared content-addressed values partition, reused across resets. */
|
|
24
|
-
values
|
|
25
|
+
values,
|
|
26
|
+
/** Whether close() should release the underlying keyspace. */
|
|
27
|
+
ownsRoot) {
|
|
25
28
|
this.root = root;
|
|
26
29
|
this.values = values;
|
|
30
|
+
this.ownsRoot = ownsRoot;
|
|
27
31
|
}
|
|
28
32
|
/** Open (or create) the keyspace at `dbPath` and its `values` partition. */
|
|
29
33
|
static async open(dbPath, options = {}) {
|
|
@@ -33,13 +37,18 @@ export class FjallValuesSession {
|
|
|
33
37
|
const root = await FjallRoot.open(dbPath, options);
|
|
34
38
|
try {
|
|
35
39
|
const values = await root.writablePartition("values");
|
|
36
|
-
return new FjallValuesSession(root, values);
|
|
40
|
+
return new FjallValuesSession(root, values, true);
|
|
37
41
|
}
|
|
38
42
|
catch (e) {
|
|
39
43
|
await root.close();
|
|
40
44
|
throw e;
|
|
41
45
|
}
|
|
42
46
|
}
|
|
47
|
+
/** Wrap an already-open keyspace and open its `values` partition. */
|
|
48
|
+
static async fromRoot(root) {
|
|
49
|
+
const values = await root.writablePartition("values");
|
|
50
|
+
return new FjallValuesSession(root, values, false);
|
|
51
|
+
}
|
|
43
52
|
/** Flush the journal to disk (a no-op for ephemeral keyspaces). */
|
|
44
53
|
async persist() {
|
|
45
54
|
await this.root.persist();
|
|
@@ -50,7 +59,9 @@ export class FjallValuesSession {
|
|
|
50
59
|
}
|
|
51
60
|
/** Release the keyspace handle (skips the sync-all fsync when ephemeral). */
|
|
52
61
|
async close() {
|
|
53
|
-
|
|
62
|
+
if (this.ownsRoot) {
|
|
63
|
+
await this.root.close();
|
|
64
|
+
}
|
|
54
65
|
}
|
|
55
66
|
}
|
|
56
67
|
/**
|
|
@@ -67,7 +78,7 @@ export class FjallValuesSession {
|
|
|
67
78
|
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
68
79
|
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
69
80
|
* anyway (the leaf sets live in memory), so values left over by a previous run
|
|
70
|
-
* are never collected. An instance backed by a shared
|
|
81
|
+
* are never collected. An instance backed by a shared keyspace (fuzz reset
|
|
71
82
|
* reuse) only ever prunes values it inserted itself, since its refcounts start
|
|
72
83
|
* empty - values left behind by earlier resets stay untouched.
|
|
73
84
|
*/
|
|
@@ -81,6 +92,11 @@ export class HybridSerializedStates {
|
|
|
81
92
|
// This instance owns the session it just opened, so its `close()` closes it.
|
|
82
93
|
return new HybridSerializedStates(spec, blake2b, session, true);
|
|
83
94
|
}
|
|
95
|
+
/** Create a db over an already-open keyspace owned by the caller. */
|
|
96
|
+
static async fromRoot(spec, blake2b, root) {
|
|
97
|
+
const session = await FjallValuesSession.fromRoot(root);
|
|
98
|
+
return new HybridSerializedStates(spec, blake2b, session, true);
|
|
99
|
+
}
|
|
84
100
|
/**
|
|
85
101
|
* Wrap an already-open `FjallValuesSession` and reuse its keyspace.
|
|
86
102
|
*
|
|
@@ -43,6 +43,8 @@ export declare class FjallRoot {
|
|
|
43
43
|
partition(name: string): Promise<FjallPartition>;
|
|
44
44
|
/** Open a writable partition, failing early when this root is read-only. */
|
|
45
45
|
writablePartition(name: string): Promise<Partition>;
|
|
46
|
+
/** Delete a writable partition and all of its data. */
|
|
47
|
+
deletePartition(name: string): Promise<void>;
|
|
46
48
|
/**
|
|
47
49
|
* Flush the journal to disk so prior writes survive a crash.
|
|
48
50
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,SAAS,EAEd,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAIzB,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC;AAC7C,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;AAE3D;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAKpE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,SAAS;IAElB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO;IAOP,2DAA2D;WAC9C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,SAAS,CAAC;IAWrF,sEAAsE;IACtE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,wDAAwD;IAClD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAItD,4EAA4E;IACtE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAOzD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAU9B;;;;;;OAMG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,oFAAoF;IAC9E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
1
|
+
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,SAAS,EAEd,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAIzB,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC;AAC7C,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;AAE3D;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAKpE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,SAAS;IAElB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO;IAOP,2DAA2D;WAC9C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,SAAS,CAAC;IAWrF,sEAAsE;IACtE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,wDAAwD;IAClD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAItD,4EAA4E;IACtE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAOzD,uDAAuD;IACjD,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAU9B;;;;;;OAMG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,oFAAoF;IAC9E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -31,7 +31,7 @@ export class FjallRoot {
|
|
|
31
31
|
}
|
|
32
32
|
/** Open (or create) a fjall keyspace at the given path. */
|
|
33
33
|
static async open(dbPath, options = {}) {
|
|
34
|
-
// fjall-js
|
|
34
|
+
// fjall-js shares one engine per path. Readers must use the read-only
|
|
35
35
|
// wrapper surface; durability is driven explicitly through persist().
|
|
36
36
|
const config = {
|
|
37
37
|
path: dbPath,
|
|
@@ -55,6 +55,13 @@ export class FjallRoot {
|
|
|
55
55
|
}
|
|
56
56
|
return (await this.keyspace.partition(name));
|
|
57
57
|
}
|
|
58
|
+
/** Delete a writable partition and all of its data. */
|
|
59
|
+
async deletePartition(name) {
|
|
60
|
+
if (this.readOnly) {
|
|
61
|
+
throw new Error(`Cannot delete fjall partition '${name}' from a read-only keyspace.`);
|
|
62
|
+
}
|
|
63
|
+
await this.keyspace.deletePartition(name);
|
|
64
|
+
}
|
|
58
65
|
/**
|
|
59
66
|
* Flush the journal to disk so prior writes survive a crash.
|
|
60
67
|
*
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"root.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { describe, it } from "node:test";
|
|
4
|
+
import { FjallRoot } from "./root.js";
|
|
5
|
+
const key = Buffer.from("key");
|
|
6
|
+
const value = Buffer.from("value");
|
|
7
|
+
describe("FjallRoot", () => {
|
|
8
|
+
it("deletes a partition and recreates it empty", async () => {
|
|
9
|
+
const dbPath = fs.mkdtempSync("typeberry-fjall-root-");
|
|
10
|
+
const root = await FjallRoot.open(dbPath, { ephemeral: true });
|
|
11
|
+
try {
|
|
12
|
+
const first = await root.writablePartition("scratch");
|
|
13
|
+
await first.insert(key, value);
|
|
14
|
+
await root.deletePartition("scratch");
|
|
15
|
+
const second = await root.writablePartition("scratch");
|
|
16
|
+
assert.strictEqual(second.get(key), null);
|
|
17
|
+
}
|
|
18
|
+
finally {
|
|
19
|
+
await root.close();
|
|
20
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
});
|
|
@@ -16,6 +16,7 @@ export declare class FjallStates implements StatesDb<SerializedState<LeafDb>>, I
|
|
|
16
16
|
private readonly values;
|
|
17
17
|
static open(spec: ChainSpec, blake2b: Blake2b, root: FjallRoot): Promise<FjallStates>;
|
|
18
18
|
private readonly valuesDb;
|
|
19
|
+
private pendingPrune;
|
|
19
20
|
private constructor();
|
|
20
21
|
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
21
22
|
updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAE/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAI9F,0DAA0D;AAC1D,qBAAa,WAAY,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAE/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAuB,KAAK,SAAS,EAAgC,MAAM,WAAW,CAAC;AAI9F,0DAA0D;AAC1D,qBAAa,WAAY,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAU7F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;WAbZ,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAK3F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,YAAY,CAAuC;IAE3D,OAAO;IAUD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAQxG,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAUlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAahE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAK7C,UAAU,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMxC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;YAIG,eAAe;IAyB7B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -19,6 +19,7 @@ export class FjallStates {
|
|
|
19
19
|
return new FjallStates(spec, blake2b, root, states, values);
|
|
20
20
|
}
|
|
21
21
|
valuesDb;
|
|
22
|
+
pendingPrune = Promise.resolve();
|
|
22
23
|
constructor(spec, blake2b, root, states, values) {
|
|
23
24
|
this.spec = spec;
|
|
24
25
|
this.blake2b = blake2b;
|
|
@@ -58,14 +59,16 @@ export class FjallStates {
|
|
|
58
59
|
// would need counts persisted (and crash-consistent) alongside the values.
|
|
59
60
|
}
|
|
60
61
|
markUnused(headerHash) {
|
|
61
|
-
|
|
62
|
-
.remove(headerHash.raw)
|
|
62
|
+
this.pendingPrune = this.pendingPrune
|
|
63
|
+
.then(() => writable(this.states, this.root).remove(headerHash.raw))
|
|
63
64
|
.catch((e) => logger.warn `Failed to prune state ${headerHash}: ${e}`);
|
|
64
65
|
}
|
|
65
66
|
diskSizeInBytes() {
|
|
66
67
|
return this.root.sizeInBytes();
|
|
67
68
|
}
|
|
68
|
-
async close() {
|
|
69
|
+
async close() {
|
|
70
|
+
await this.pendingPrune;
|
|
71
|
+
}
|
|
69
72
|
async updateAndCommit(headerHash, leafs, data) {
|
|
70
73
|
const { values } = updateLeafs(leafs, this.blake2b, data);
|
|
71
74
|
const stateLeafs = BytesBlob.blobFromParts(leafs.array.map((x) => x.node.raw));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA+MrH"}
|
|
@@ -8,7 +8,7 @@ import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
|
|
|
8
8
|
import { HASH_SIZE } from "#@typeberry/hash";
|
|
9
9
|
import { Logger } from "#@typeberry/logger";
|
|
10
10
|
import { CURRENT_VERSION, Result, version } from "#@typeberry/utils";
|
|
11
|
-
import {
|
|
11
|
+
import { FjallRoot, logHostEnvironment } from "#@typeberry/workers-api-node";
|
|
12
12
|
import { getChainSpec } from "./common.js";
|
|
13
13
|
import { mainImporter } from "./main-importer.js";
|
|
14
14
|
const logger = Logger.new(import.meta.filename, "fuzztarget");
|
|
@@ -16,16 +16,15 @@ const logger = Logger.new(import.meta.filename, "fuzztarget");
|
|
|
16
16
|
const FUZZ_DB_SUBDIR = "typeberry-fuzz-db";
|
|
17
17
|
const FUZZ_DB_FJALL = "fjall-hybrid";
|
|
18
18
|
const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, "fjall"];
|
|
19
|
-
/**
|
|
20
|
-
const
|
|
19
|
+
/** The partitions used by FjallBlocks and FjallStates in the full-fjall backend. */
|
|
20
|
+
const FUZZ_FJALL_PARTITIONS = ["headers", "extrinsics", "postStateRoots", "states", "values"];
|
|
21
21
|
/**
|
|
22
|
-
* Size of the fjall block-cache for the fuzz session.
|
|
23
|
-
* resets
|
|
24
|
-
* the resident memory bounded.
|
|
22
|
+
* Size of the fjall block-cache for the fuzz session. The keyspace stays open
|
|
23
|
+
* across resets, so this cache is what keeps the resident memory bounded.
|
|
25
24
|
*/
|
|
26
25
|
const FUZZ_FJALL_CACHE_BYTES = 128 * 1024 * 1024;
|
|
27
|
-
/** Rebuild
|
|
28
|
-
const
|
|
26
|
+
/** Rebuild reused fjall keyspaces every N resets to limit LSM write/read amplification. */
|
|
27
|
+
const REBUILD_FJALL_KEYSPACE_EVERY = 50;
|
|
29
28
|
/**
|
|
30
29
|
* Resolve the directory the fuzzer should use for its on-disk database, or
|
|
31
30
|
* `undefined` for an in-memory database. The dedicated `FUZZ_DB_SUBDIR` is
|
|
@@ -71,11 +70,10 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
71
70
|
logger.info `🗄️ Fuzz persistent backend: ${hybridStateBackend}.`;
|
|
72
71
|
}
|
|
73
72
|
let runningNode = null;
|
|
74
|
-
// The fjall
|
|
75
|
-
//
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
// Track how many times resetState has been called for periodic fjall session rebuilds.
|
|
73
|
+
// The fjall keyspace is opened once per fuzz session and reused on every
|
|
74
|
+
// reset, because opening it is the slow part.
|
|
75
|
+
let fjallKeyspace = null;
|
|
76
|
+
// Track how many times resetState has been called for periodic fjall keyspace rebuilds.
|
|
79
77
|
let resetCount = 0;
|
|
80
78
|
// Set when close() starts. Guards resetState so a fuzz command arriving
|
|
81
79
|
// mid-shutdown can't build a fresh node that close() then orphans.
|
|
@@ -143,45 +141,67 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
143
141
|
// db too much. Tiny stays uncompressed, its db is small and speed matters more.
|
|
144
142
|
ephemeral: isPersistent,
|
|
145
143
|
stateBackend: isPersistent ? hybridStateBackend : "fjall",
|
|
146
|
-
// Reuse the
|
|
147
|
-
//
|
|
148
|
-
|
|
144
|
+
// Reuse the keyspace for both fjall backends. Nothing to pass for
|
|
145
|
+
// the in-memory fallback.
|
|
146
|
+
sharedFjallKeyspace: fjallKeyspace ?? undefined,
|
|
149
147
|
});
|
|
150
148
|
};
|
|
151
149
|
if (fuzzDbBase !== undefined) {
|
|
152
150
|
try {
|
|
151
|
+
const fjallKeyspacePath = withRelPath(fuzzDbBase);
|
|
153
152
|
if (hybridStateBackend === FUZZ_DB_FJALL) {
|
|
154
|
-
// fjall-hybrid:
|
|
155
|
-
//
|
|
156
|
-
|
|
157
|
-
if (resetCount === 1) {
|
|
153
|
+
// fjall-hybrid: values pile up across resets, so rebuild the
|
|
154
|
+
// keyspace periodically to avoid LSM read amplification.
|
|
155
|
+
if (resetCount === 1 || fjallKeyspace === null) {
|
|
158
156
|
// First reset: start from a clean slate.
|
|
159
157
|
await wipeFuzzDb(fuzzDbBase);
|
|
160
|
-
|
|
158
|
+
fjallKeyspace = await FjallRoot.open(fjallKeyspacePath, {
|
|
161
159
|
ephemeral: true,
|
|
162
160
|
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
163
161
|
});
|
|
164
|
-
logger.info `🗄️ Opened reusable fjall
|
|
162
|
+
logger.info `🗄️ Opened reusable fjall keyspace at ${fjallKeyspacePath}`;
|
|
165
163
|
}
|
|
166
|
-
if (resetCount %
|
|
167
|
-
// Periodic rebuild: close, wipe
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
await
|
|
171
|
-
await wipeFuzzDb(
|
|
172
|
-
|
|
173
|
-
if (fjallSession === null) {
|
|
174
|
-
// No active session: create a fresh one.
|
|
175
|
-
fjallSession = await FjallValuesSession.open(fjallSessionPath, {
|
|
164
|
+
else if (resetCount % REBUILD_FJALL_KEYSPACE_EVERY === 0) {
|
|
165
|
+
// Periodic rebuild: close, wipe keyspace dir, and reopen.
|
|
166
|
+
const keyspace = fjallKeyspace;
|
|
167
|
+
fjallKeyspace = null;
|
|
168
|
+
await keyspace.close().catch(() => { });
|
|
169
|
+
await wipeFuzzDb(fuzzDbBase).catch(() => { });
|
|
170
|
+
fjallKeyspace = await FjallRoot.open(fjallKeyspacePath, {
|
|
176
171
|
ephemeral: true,
|
|
177
172
|
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
178
173
|
});
|
|
179
|
-
logger.info `🗄️
|
|
174
|
+
logger.info `🗄️ Rebuilt reusable fjall keyspace at ${fjallKeyspacePath}`;
|
|
180
175
|
}
|
|
181
176
|
}
|
|
182
177
|
else {
|
|
183
|
-
//
|
|
184
|
-
|
|
178
|
+
// full-fjall: keep one keyspace open and recycle only the five
|
|
179
|
+
// partitions used by FjallBlocks/FjallStates.
|
|
180
|
+
if (resetCount === 1 || fjallKeyspace === null) {
|
|
181
|
+
await wipeFuzzDb(fuzzDbBase);
|
|
182
|
+
fjallKeyspace = await FjallRoot.open(fjallKeyspacePath, {
|
|
183
|
+
ephemeral: true,
|
|
184
|
+
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
185
|
+
});
|
|
186
|
+
logger.info `🗄️ Opened reusable fjall keyspace at ${fjallKeyspacePath}`;
|
|
187
|
+
}
|
|
188
|
+
else if (resetCount % REBUILD_FJALL_KEYSPACE_EVERY === 0) {
|
|
189
|
+
// Periodic rebuild: delete/recreate keeps correctness, but a
|
|
190
|
+
// long-lived keyspace accumulates fjall write amplification.
|
|
191
|
+
const keyspace = fjallKeyspace;
|
|
192
|
+
fjallKeyspace = null;
|
|
193
|
+
await keyspace.close().catch(() => { });
|
|
194
|
+
await wipeFuzzDb(fuzzDbBase).catch(() => { });
|
|
195
|
+
fjallKeyspace = await FjallRoot.open(fjallKeyspacePath, {
|
|
196
|
+
ephemeral: true,
|
|
197
|
+
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
198
|
+
});
|
|
199
|
+
logger.info `🗄️ Rebuilt reusable fjall keyspace at ${fjallKeyspacePath}`;
|
|
200
|
+
}
|
|
201
|
+
else if (fjallKeyspace !== null) {
|
|
202
|
+
const keyspace = fjallKeyspace;
|
|
203
|
+
await Promise.all(FUZZ_FJALL_PARTITIONS.map((name) => keyspace.deletePartition(name)));
|
|
204
|
+
}
|
|
185
205
|
}
|
|
186
206
|
runningNode = await buildNode(fuzzDbBase);
|
|
187
207
|
return await runningNode.getBestStateRootHash();
|
|
@@ -210,9 +230,9 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
210
230
|
isClosing = true;
|
|
211
231
|
// Stop accepting connections + unlink the socket.
|
|
212
232
|
closeFuzzTarget();
|
|
213
|
-
// Drain the active
|
|
233
|
+
// Drain the active reset (flush + close DB). Swallow errors so a
|
|
214
234
|
// failing close still lets the process exit 0; the db is wiped next.
|
|
215
|
-
// The node references the shared fjall
|
|
235
|
+
// The node references the shared fjall keyspace, so it must close first.
|
|
216
236
|
if (activeReset !== null) {
|
|
217
237
|
await activeReset.catch((e) => logger.error `Error waiting for fuzz reset: ${e}`);
|
|
218
238
|
}
|
|
@@ -221,11 +241,11 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
221
241
|
runningNode = null;
|
|
222
242
|
await node.close().catch((e) => logger.error `Error closing fuzz node: ${e}`);
|
|
223
243
|
}
|
|
224
|
-
// Release the reused fjall
|
|
225
|
-
if (
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
await
|
|
244
|
+
// Release the reused fjall keyspace before wiping its files.
|
|
245
|
+
if (fjallKeyspace !== null) {
|
|
246
|
+
const keyspace = fjallKeyspace;
|
|
247
|
+
fjallKeyspace = null;
|
|
248
|
+
await keyspace.close().catch((e) => logger.error `Error closing fjall keyspace: ${e}`);
|
|
229
249
|
}
|
|
230
250
|
if (fuzzDbBase !== undefined) {
|
|
231
251
|
await wipeFuzzDb(fuzzDbBase).catch(() => { });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type
|
|
1
|
+
import { type FjallRoot } from "#@typeberry/workers-api-node";
|
|
2
2
|
import type { JamConfig } from "./jam-config.js";
|
|
3
3
|
import type { NodeApi } from "./main.js";
|
|
4
4
|
export type StateBackend = "fjall" | "fjall-hybrid";
|
|
@@ -11,11 +11,10 @@ export type ImporterOptions = {
|
|
|
11
11
|
/** Persistent backend used when `databaseBasePath` is set. Defaults to fjall. */
|
|
12
12
|
stateBackend?: StateBackend;
|
|
13
13
|
/**
|
|
14
|
-
* Reuse an already-open fjall
|
|
15
|
-
*
|
|
16
|
-
* opens one per run and reuses it across resets.
|
|
14
|
+
* Reuse an already-open fjall keyspace instead of opening a fresh keyspace
|
|
15
|
+
* from `dbPath`. Used by fjall and fjall-hybrid fuzz targets.
|
|
17
16
|
*/
|
|
18
|
-
|
|
17
|
+
sharedFjallKeyspace?: FjallRoot;
|
|
19
18
|
};
|
|
20
19
|
export declare function mainImporter(config: JamConfig, withRelPath: (v: string) => string, options?: ImporterOptions): Promise<NodeApi>;
|
|
21
20
|
//# sourceMappingURL=main-importer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,
|
|
1
|
+
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,SAAS,EAA4D,MAAM,6BAA6B,CAAC;AAEvH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,cAAc,CAAC;AAEpD,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,iFAAiF;IACjF,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;CACjC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAqGlB"}
|
|
@@ -6,7 +6,7 @@ import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
|
6
6
|
import { createImporter, ImporterConfig } from "#@typeberry/importer";
|
|
7
7
|
import { tryAsU16 } from "#@typeberry/numbers";
|
|
8
8
|
import { CURRENT_SUITE, CURRENT_VERSION, Result, resultToString, version } from "#@typeberry/utils";
|
|
9
|
-
import { FjallWorkerConfig, HybridWorkerConfig, InMemWorkerConfig
|
|
9
|
+
import { FjallWorkerConfig, HybridWorkerConfig, InMemWorkerConfig } from "#@typeberry/workers-api-node";
|
|
10
10
|
import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
|
|
11
11
|
const zeroHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
12
12
|
export async function mainImporter(config, withRelPath, options = {}) {
|
|
@@ -48,7 +48,7 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
48
48
|
workerParams,
|
|
49
49
|
ephemeral,
|
|
50
50
|
compression,
|
|
51
|
-
|
|
51
|
+
sharedFjallKeyspace: options.sharedFjallKeyspace,
|
|
52
52
|
})
|
|
53
53
|
: FjallWorkerConfig.new({
|
|
54
54
|
nodeName,
|
|
@@ -57,6 +57,7 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
57
57
|
dbPath,
|
|
58
58
|
workerParams,
|
|
59
59
|
ephemeral,
|
|
60
|
+
sharedFjallKeyspace: options.sharedFjallKeyspace,
|
|
60
61
|
});
|
|
61
62
|
// Initialize the database with genesis state and block if there isn't one.
|
|
62
63
|
logger.info `🛢️ Opening database at ${dbPath}`;
|
|
@@ -2,11 +2,11 @@ import type { MessagePort } from "node:worker_threads";
|
|
|
2
2
|
import { type Decode, type Encode } from "#@typeberry/codec";
|
|
3
3
|
import { ChainSpec } from "#@typeberry/config";
|
|
4
4
|
import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
|
|
5
|
-
import { FjallValuesSession } from "#@typeberry/database-fjall";
|
|
5
|
+
import { FjallRoot, FjallValuesSession } from "#@typeberry/database-fjall";
|
|
6
6
|
import { Blake2b } from "#@typeberry/hash";
|
|
7
7
|
import type { WorkerConfig } from "#@typeberry/workers-api";
|
|
8
8
|
import { ThreadPort, type TransferablePort } from "./port.js";
|
|
9
|
-
export { FjallValuesSession };
|
|
9
|
+
export { FjallRoot, FjallValuesSession };
|
|
10
10
|
/** Persistent regular-node backend. */
|
|
11
11
|
export type PersistentBackend = "fjall";
|
|
12
12
|
/** Transferable worker config for persistent regular-node workers. */
|
|
@@ -21,7 +21,8 @@ export declare class FjallWorkerConfig<T = void> implements WorkerConfig<T, Bloc
|
|
|
21
21
|
readonly ports: Map<string, ThreadPort>;
|
|
22
22
|
readonly ephemeral: boolean;
|
|
23
23
|
readonly cacheSizeBytes: number | undefined;
|
|
24
|
-
|
|
24
|
+
private readonly sharedFjallKeyspace;
|
|
25
|
+
static new<T>({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes, sharedFjallKeyspace, }: {
|
|
25
26
|
nodeName: string;
|
|
26
27
|
chainSpec: ChainSpec;
|
|
27
28
|
workerParams: T;
|
|
@@ -30,6 +31,7 @@ export declare class FjallWorkerConfig<T = void> implements WorkerConfig<T, Bloc
|
|
|
30
31
|
ports?: Map<string, ThreadPort>;
|
|
31
32
|
ephemeral?: boolean;
|
|
32
33
|
cacheSizeBytes?: number;
|
|
34
|
+
sharedFjallKeyspace?: FjallRoot;
|
|
33
35
|
}): FjallWorkerConfig<T>;
|
|
34
36
|
/** Restore node config from a transferable config object. */
|
|
35
37
|
static fromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<FjallWorkerConfig<T>>;
|
|
@@ -109,7 +111,7 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
|
|
|
109
111
|
readonly ephemeral: boolean;
|
|
110
112
|
readonly compression: boolean;
|
|
111
113
|
private readonly states;
|
|
112
|
-
static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression,
|
|
114
|
+
static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, sharedFjallKeyspace, }: {
|
|
113
115
|
nodeName: string;
|
|
114
116
|
chainSpec: ChainSpec;
|
|
115
117
|
workerParams: T;
|
|
@@ -118,11 +120,11 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
|
|
|
118
120
|
ephemeral?: boolean;
|
|
119
121
|
compression?: boolean;
|
|
120
122
|
/**
|
|
121
|
-
* Reuse an already-open fjall
|
|
122
|
-
*
|
|
123
|
-
*
|
|
123
|
+
* Reuse an already-open fjall keyspace instead of opening a fresh keyspace.
|
|
124
|
+
* The fuzz target opens one per run and passes it on every reset, so only
|
|
125
|
+
* the in-memory blocks/leaf sets are rebuilt per vector.
|
|
124
126
|
*/
|
|
125
|
-
|
|
127
|
+
sharedFjallKeyspace?: FjallRoot;
|
|
126
128
|
}): Promise<HybridWorkerConfig<T>>;
|
|
127
129
|
private readonly blocks;
|
|
128
130
|
private constructor();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAGL,SAAS,EAET,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAI9D,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC;AAEzC,uCAAuC;AACvC,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAExC,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAE7D,kEAAkE;AAClE,qBAAa,iBAAiB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAsD7E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAE9B,SAAS,EAAE,OAAO;aAClB,cAAc,EAAE,MAAM,GAAG,SAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IA9DtC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,EACjB,cAAc,EACd,mBAAmB,GACpB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,mBAAmB,CAAC,EAAE,SAAS,CAAC;KACjC;IAcD,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAiBpF,OAAO;IAaD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAiDhD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAW7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,EAAE,iBAAiB,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAgBF,qEAAqE;AACrE,wBAAsB,gCAAgC,CAAC,CAAC,EACtD,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EACvB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAEpC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CAQjD;AAED,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC;AAEpC;;;;;;;;;;;;;;GAcG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAqCnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;aAClB,WAAW,EAAE,OAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;WA3CZ,GAAG,CAAC,CAAC,EAAE,EAClB,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,EACjB,WAAkB,EAClB,mBAAmB,GACpB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB;;;;WAIG;QACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;KACjC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAUlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IAExC,OAAO;IAaD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CASjD"}
|
|
@@ -4,9 +4,9 @@ import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database"
|
|
|
4
4
|
import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
|
|
5
5
|
import { Blake2b } from "#@typeberry/hash";
|
|
6
6
|
import { ThreadPort } from "./port.js";
|
|
7
|
-
// Re-exported so the fuzz target can open one
|
|
8
|
-
//
|
|
9
|
-
export { FjallValuesSession };
|
|
7
|
+
// Re-exported so the fuzz target can open one keyspace per run and reuse it
|
|
8
|
+
// across resets.
|
|
9
|
+
export { FjallRoot, FjallValuesSession };
|
|
10
10
|
/** Worker config for node.js, backed by a shared fjall engine. */
|
|
11
11
|
export class FjallWorkerConfig {
|
|
12
12
|
nodeName;
|
|
@@ -17,8 +17,9 @@ export class FjallWorkerConfig {
|
|
|
17
17
|
ports;
|
|
18
18
|
ephemeral;
|
|
19
19
|
cacheSizeBytes;
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
sharedFjallKeyspace;
|
|
21
|
+
static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, sharedFjallKeyspace, }) {
|
|
22
|
+
return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes, sharedFjallKeyspace);
|
|
22
23
|
}
|
|
23
24
|
/** Restore node config from a transferable config object. */
|
|
24
25
|
static async fromTransferable(decodeParams, config) {
|
|
@@ -38,7 +39,7 @@ export class FjallWorkerConfig {
|
|
|
38
39
|
}
|
|
39
40
|
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
40
41
|
// Kept for the fuzz/importer path. When set, persist() is skipped.
|
|
41
|
-
ephemeral = false, cacheSizeBytes = undefined) {
|
|
42
|
+
ephemeral = false, cacheSizeBytes = undefined, sharedFjallKeyspace = undefined) {
|
|
42
43
|
this.nodeName = nodeName;
|
|
43
44
|
this.chainSpec = chainSpec;
|
|
44
45
|
this.workerParams = workerParams;
|
|
@@ -47,13 +48,19 @@ export class FjallWorkerConfig {
|
|
|
47
48
|
this.ports = ports;
|
|
48
49
|
this.ephemeral = ephemeral;
|
|
49
50
|
this.cacheSizeBytes = cacheSizeBytes;
|
|
51
|
+
this.sharedFjallKeyspace = sharedFjallKeyspace;
|
|
50
52
|
}
|
|
51
53
|
async openDatabase(options = { readonly: true }) {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
54
|
+
if (this.sharedFjallKeyspace !== undefined && options.readonly) {
|
|
55
|
+
throw new Error("Cannot open a read-only fjall database from a shared writable keyspace.");
|
|
56
|
+
}
|
|
57
|
+
const fjall = this.sharedFjallKeyspace ??
|
|
58
|
+
(await FjallRoot.open(this.dbPath, {
|
|
59
|
+
readOnly: options.readonly,
|
|
60
|
+
ephemeral: this.ephemeral,
|
|
61
|
+
cacheSizeBytes: this.cacheSizeBytes,
|
|
62
|
+
}));
|
|
63
|
+
const ownsFjall = this.sharedFjallKeyspace === undefined;
|
|
57
64
|
let blocks = null;
|
|
58
65
|
let states = null;
|
|
59
66
|
try {
|
|
@@ -63,7 +70,9 @@ export class FjallWorkerConfig {
|
|
|
63
70
|
]);
|
|
64
71
|
}
|
|
65
72
|
catch (e) {
|
|
66
|
-
|
|
73
|
+
if (ownsFjall) {
|
|
74
|
+
await fjall.close();
|
|
75
|
+
}
|
|
67
76
|
throw e;
|
|
68
77
|
}
|
|
69
78
|
return {
|
|
@@ -82,7 +91,9 @@ export class FjallWorkerConfig {
|
|
|
82
91
|
close: async () => {
|
|
83
92
|
blocks = null;
|
|
84
93
|
states = null;
|
|
85
|
-
|
|
94
|
+
if (ownsFjall) {
|
|
95
|
+
await fjall.close();
|
|
96
|
+
}
|
|
86
97
|
},
|
|
87
98
|
};
|
|
88
99
|
}
|
|
@@ -181,11 +192,11 @@ export class HybridWorkerConfig {
|
|
|
181
192
|
ephemeral;
|
|
182
193
|
compression;
|
|
183
194
|
states;
|
|
184
|
-
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true,
|
|
195
|
+
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, sharedFjallKeyspace, }) {
|
|
185
196
|
// The values store is created once here and shared across reopen. When a
|
|
186
|
-
//
|
|
187
|
-
const states =
|
|
188
|
-
? FjallHybridSerializedStates.
|
|
197
|
+
// keyspace is given (fuzz reset reuse) we wrap it instead of opening a new one.
|
|
198
|
+
const states = sharedFjallKeyspace !== undefined
|
|
199
|
+
? await FjallHybridSerializedStates.fromRoot(chainSpec, blake2b, sharedFjallKeyspace)
|
|
189
200
|
: await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral });
|
|
190
201
|
return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
|
|
191
202
|
}
|
|
@@ -4,6 +4,7 @@ import { describe, it } from "node:test";
|
|
|
4
4
|
import { Bytes } from "#@typeberry/bytes";
|
|
5
5
|
import { codec } from "#@typeberry/codec";
|
|
6
6
|
import { tinyChainSpec } from "#@typeberry/config";
|
|
7
|
+
import { FjallRoot } from "#@typeberry/database-fjall";
|
|
7
8
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
8
9
|
import { tryAsU32 } from "#@typeberry/numbers";
|
|
9
10
|
import { configTransferList, FjallWorkerConfig, HybridWorkerConfig } from "./config.js";
|
|
@@ -56,6 +57,30 @@ describe("FjallWorkerConfig transfer list", () => {
|
|
|
56
57
|
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
57
58
|
}
|
|
58
59
|
});
|
|
60
|
+
it("can borrow a shared fjall root without closing it", async () => {
|
|
61
|
+
const blake2b = await Blake2b.createHasher();
|
|
62
|
+
const dbPath = fs.mkdtempSync("typeberry-fjall-worker-shared-");
|
|
63
|
+
const root = await FjallRoot.open(dbPath, { ephemeral: true });
|
|
64
|
+
const config = FjallWorkerConfig.new({
|
|
65
|
+
nodeName: "node",
|
|
66
|
+
chainSpec: spec,
|
|
67
|
+
workerParams: undefined,
|
|
68
|
+
dbPath,
|
|
69
|
+
blake2b,
|
|
70
|
+
sharedFjallKeyspace: root,
|
|
71
|
+
});
|
|
72
|
+
try {
|
|
73
|
+
const db = await config.openDatabase({ readonly: false });
|
|
74
|
+
await db.close();
|
|
75
|
+
await root.deletePartition("headers");
|
|
76
|
+
const headers = await root.writablePartition("headers");
|
|
77
|
+
assert.strictEqual(headers.get(Bytes.zero(HASH_SIZE).raw), null);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await root.close();
|
|
81
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
82
|
+
}
|
|
83
|
+
});
|
|
59
84
|
});
|
|
60
85
|
describe("HybridWorkerConfig", () => {
|
|
61
86
|
it("constructs and opens a fjall-backed hybrid db", async () => {
|
|
@@ -87,4 +112,35 @@ describe("HybridWorkerConfig", () => {
|
|
|
87
112
|
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
88
113
|
}
|
|
89
114
|
});
|
|
115
|
+
it("can borrow a shared fjall root for the values store", async () => {
|
|
116
|
+
const blake2b = await Blake2b.createHasher();
|
|
117
|
+
const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-shared-");
|
|
118
|
+
const root = await FjallRoot.open(dbPath, { ephemeral: true });
|
|
119
|
+
try {
|
|
120
|
+
const config = await HybridWorkerConfig.new({
|
|
121
|
+
nodeName: "node",
|
|
122
|
+
chainSpec: spec,
|
|
123
|
+
workerParams: undefined,
|
|
124
|
+
blake2b,
|
|
125
|
+
dbPath,
|
|
126
|
+
ephemeral: true,
|
|
127
|
+
sharedFjallKeyspace: root,
|
|
128
|
+
});
|
|
129
|
+
const db = await config.openDatabase({ readonly: false });
|
|
130
|
+
const states = db.getStatesDb();
|
|
131
|
+
try {
|
|
132
|
+
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
133
|
+
assert.notStrictEqual(states, undefined);
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
await states.close();
|
|
137
|
+
await db.close();
|
|
138
|
+
}
|
|
139
|
+
await root.deletePartition("values");
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
await root.close();
|
|
143
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
144
|
+
}
|
|
145
|
+
});
|
|
90
146
|
});
|