@typeberry/lib 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/core/collections/sorted-array.d.ts +1 -1
- package/packages/core/collections/sorted-array.d.ts.map +1 -1
- package/packages/core/collections/sorted-array.js +4 -2
- package/packages/core/collections/sorted-array.test.js +10 -0
- package/packages/core/collections/sorted-set.d.ts +1 -1
- package/packages/core/collections/sorted-set.d.ts.map +1 -1
- package/packages/core/collections/sorted-set.js +6 -2
- package/packages/core/collections/sorted-set.test.js +24 -0
- package/packages/core/utils/index.d.ts +1 -0
- package/packages/core/utils/index.d.ts.map +1 -1
- package/packages/core/utils/index.js +1 -0
- package/packages/core/utils/shutdown.d.ts +18 -0
- package/packages/core/utils/shutdown.d.ts.map +1 -0
- package/packages/core/utils/shutdown.js +3 -0
- package/packages/core/utils/shutdown.node.d.ts +5 -0
- package/packages/core/utils/shutdown.node.d.ts.map +1 -0
- package/packages/core/utils/shutdown.node.js +76 -0
- package/packages/core/utils/shutdown.test.d.ts +2 -0
- package/packages/core/utils/shutdown.test.d.ts.map +1 -0
- package/packages/core/utils/shutdown.test.js +164 -0
- package/packages/extensions/ipc/server.d.ts.map +1 -1
- package/packages/extensions/ipc/server.js +7 -0
- package/packages/jam/config-node/node-config.d.ts +20 -2
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +25 -3
- package/packages/jam/config-node/node-config.test.js +8 -1
- package/packages/jam/database/index.d.ts +1 -0
- package/packages/jam/database/index.d.ts.map +1 -1
- package/packages/jam/database/index.js +1 -0
- package/packages/jam/database/leaf-db-update.d.ts +1 -0
- package/packages/jam/database/leaf-db-update.d.ts.map +1 -1
- package/packages/jam/database/leaf-db-update.js +22 -5
- package/packages/jam/database/leaf-db-update.test.d.ts +2 -0
- package/packages/jam/database/leaf-db-update.test.d.ts.map +1 -0
- package/packages/jam/database/leaf-db-update.test.js +81 -0
- package/packages/jam/database/serialized-states-db.d.ts +4 -0
- package/packages/jam/database/serialized-states-db.d.ts.map +1 -1
- package/packages/jam/database/serialized-states-db.js +18 -2
- package/packages/jam/database/serialized-states-db.test.d.ts +2 -0
- package/packages/jam/database/serialized-states-db.test.d.ts.map +1 -0
- package/packages/jam/database/serialized-states-db.test.js +109 -0
- package/packages/jam/database/states.d.ts +11 -0
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database/states.js +3 -0
- package/packages/jam/database/value-refs.d.ts +117 -0
- package/packages/jam/database/value-refs.d.ts.map +1 -0
- package/packages/jam/database/value-refs.js +206 -0
- package/packages/jam/database/value-refs.test.d.ts +2 -0
- package/packages/jam/database/value-refs.test.d.ts.map +1 -0
- package/packages/jam/database/value-refs.test.js +165 -0
- package/packages/jam/database-fjall/blocks.d.ts +25 -0
- package/packages/jam/database-fjall/blocks.d.ts.map +1 -0
- package/packages/jam/database-fjall/blocks.js +86 -0
- package/packages/jam/database-fjall/blocks.test.d.ts +2 -0
- package/packages/jam/database-fjall/blocks.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/blocks.test.js +62 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts +61 -10
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.js +131 -26
- package/packages/jam/database-fjall/hybrid-states.test.js +125 -3
- package/packages/jam/database-fjall/index.d.ts +2 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -1
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +29 -17
- package/packages/jam/database-fjall/root.d.ts.map +1 -1
- package/packages/jam/database-fjall/root.js +33 -13
- package/packages/jam/database-fjall/states.d.ts +31 -0
- package/packages/jam/database-fjall/states.d.ts.map +1 -0
- package/packages/jam/database-fjall/states.js +99 -0
- package/packages/jam/database-fjall/states.test.d.ts +2 -0
- package/packages/jam/database-fjall/states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/states.test.js +150 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +12 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/hybrid-states.js +45 -4
- package/packages/jam/database-lmdb/hybrid-states.test.js +94 -2
- package/packages/jam/database-lmdb/states.d.ts +1 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/states.js +5 -0
- package/packages/jam/node/export.d.ts.map +1 -1
- package/packages/jam/node/export.js +18 -9
- package/packages/jam/node/main-fuzz.d.ts +4 -1
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +127 -55
- package/packages/jam/node/main-importer.d.ts +11 -2
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +27 -12
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +42 -15
- package/packages/jam/node/workers.d.ts +4 -4
- package/packages/jam/node/workers.d.ts.map +1 -1
- package/packages/workers/api/config.d.ts +2 -2
- package/packages/workers/api/config.d.ts.map +1 -1
- package/packages/workers/api/config.js +1 -1
- package/packages/workers/api/port.d.ts +5 -1
- package/packages/workers/api/port.d.ts.map +1 -1
- package/packages/workers/api/port.js +56 -13
- package/packages/workers/api/port.test.js +35 -0
- package/packages/workers/api-node/config.d.ts +58 -9
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +140 -18
- package/packages/workers/api-node/config.test.js +52 -3
- package/packages/workers/api-node/port.d.ts +3 -0
- package/packages/workers/api-node/port.d.ts.map +1 -1
- package/packages/workers/api-node/port.js +31 -2
- package/packages/workers/api-node/port.test.js +17 -0
- package/packages/workers/api-node/protocol.d.ts +3 -3
- package/packages/workers/api-node/protocol.d.ts.map +1 -1
- package/packages/workers/api-node/protocol.js +18 -12
- package/packages/workers/block-authorship/main.js +1 -1
- package/packages/workers/importer/finality.d.ts +4 -0
- package/packages/workers/importer/finality.d.ts.map +1 -1
- package/packages/workers/importer/finality.js +10 -1
- package/packages/workers/importer/finality.test.js +6 -0
- package/packages/workers/importer/importer.d.ts.map +1 -1
- package/packages/workers/importer/importer.js +3 -0
- package/packages/workers/importer/main.d.ts +1 -1
- package/packages/workers/importer/main.d.ts.map +1 -1
- package/packages/workers/importer/main.js +1 -1
- package/packages/workers/jam-network/main.js +1 -1
|
@@ -6,27 +6,31 @@ import { EventEmitter } from "eventemitter3";
|
|
|
6
6
|
* so there is no need for any serialization - we simply pass the data.
|
|
7
7
|
*/
|
|
8
8
|
export class DirectPort {
|
|
9
|
-
|
|
9
|
+
inbound;
|
|
10
|
+
outbound;
|
|
10
11
|
threadId = 0;
|
|
11
12
|
/** Create a pair of symmetrical inter-connected ports. */
|
|
12
13
|
static pair() {
|
|
13
|
-
const
|
|
14
|
-
|
|
14
|
+
const left = createPortState();
|
|
15
|
+
const right = createPortState();
|
|
16
|
+
return [new DirectPort(left, right), new DirectPort(right, left)];
|
|
15
17
|
}
|
|
16
|
-
constructor(
|
|
17
|
-
this.
|
|
18
|
+
constructor(inbound, outbound) {
|
|
19
|
+
this.inbound = inbound;
|
|
20
|
+
this.outbound = outbound;
|
|
18
21
|
}
|
|
19
22
|
onClose(callback) {
|
|
20
|
-
this.events.on("error", callback);
|
|
23
|
+
this.inbound.events.on("error", callback);
|
|
21
24
|
}
|
|
22
25
|
on(event, _codec, callback) {
|
|
23
26
|
const trigger = (args) => {
|
|
24
27
|
// we simply cast the args, since there is no encoding involved.
|
|
25
28
|
callback(args);
|
|
26
29
|
};
|
|
27
|
-
this.events.on(event, trigger);
|
|
30
|
+
this.inbound.events.on(event, trigger);
|
|
31
|
+
this.flushPending(event, trigger);
|
|
28
32
|
return () => {
|
|
29
|
-
this.events.off(event, trigger);
|
|
33
|
+
this.inbound.events.off(event, trigger);
|
|
30
34
|
};
|
|
31
35
|
}
|
|
32
36
|
once(event, _codec, callback) {
|
|
@@ -34,16 +38,55 @@ export class DirectPort {
|
|
|
34
38
|
// we simply cast the args, since there is no encoding involved.
|
|
35
39
|
callback(args);
|
|
36
40
|
};
|
|
37
|
-
this.
|
|
41
|
+
const pending = this.inbound.pendingMessages.get(event);
|
|
42
|
+
const pendingMessage = pending?.shift();
|
|
43
|
+
if (pending?.length === 0) {
|
|
44
|
+
this.inbound.pendingMessages.delete(event);
|
|
45
|
+
}
|
|
46
|
+
if (pendingMessage !== undefined) {
|
|
47
|
+
trigger(pendingMessage);
|
|
48
|
+
return () => { };
|
|
49
|
+
}
|
|
50
|
+
this.inbound.events.once(event, trigger);
|
|
38
51
|
return () => {
|
|
39
|
-
this.events.off(event, trigger);
|
|
52
|
+
this.inbound.events.off(event, trigger);
|
|
40
53
|
};
|
|
41
54
|
}
|
|
42
55
|
postMessage(event, _codec, msg) {
|
|
43
|
-
this.events.
|
|
56
|
+
if (this.outbound.events.listenerCount(event) === 0) {
|
|
57
|
+
this.queuePending(event, msg);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
this.outbound.events.emit(event, msg);
|
|
44
61
|
}
|
|
45
62
|
close() {
|
|
46
|
-
this.
|
|
47
|
-
this.
|
|
63
|
+
this.closeState(this.inbound);
|
|
64
|
+
this.closeState(this.outbound);
|
|
48
65
|
}
|
|
66
|
+
queuePending(event, msg) {
|
|
67
|
+
const pending = this.outbound.pendingMessages.get(event) ?? [];
|
|
68
|
+
pending.push(msg);
|
|
69
|
+
this.outbound.pendingMessages.set(event, pending);
|
|
70
|
+
}
|
|
71
|
+
flushPending(event, trigger) {
|
|
72
|
+
const pending = this.inbound.pendingMessages.get(event);
|
|
73
|
+
if (pending === undefined) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.inbound.pendingMessages.delete(event);
|
|
77
|
+
for (const msg of pending) {
|
|
78
|
+
trigger(msg);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
closeState(state) {
|
|
82
|
+
state.events.emit("error", new Error("closing channel"));
|
|
83
|
+
state.events.removeAllListeners();
|
|
84
|
+
state.pendingMessages.clear();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function createPortState() {
|
|
88
|
+
return {
|
|
89
|
+
events: new EventEmitter(),
|
|
90
|
+
pendingMessages: new Map(),
|
|
91
|
+
};
|
|
49
92
|
}
|
|
@@ -25,4 +25,39 @@ describe("DirectPort", () => {
|
|
|
25
25
|
data: tryAsU32(10),
|
|
26
26
|
});
|
|
27
27
|
});
|
|
28
|
+
it("should deliver messages sent before the listener is attached", () => {
|
|
29
|
+
const [txPort, rxPort] = DirectPort.pair();
|
|
30
|
+
txPort.postMessage("hello", codec.u32, {
|
|
31
|
+
responseId: "10",
|
|
32
|
+
data: tryAsU32(10),
|
|
33
|
+
});
|
|
34
|
+
let receivedMessage = null;
|
|
35
|
+
rxPort.on("hello", codec.u32, (msg) => {
|
|
36
|
+
receivedMessage = msg;
|
|
37
|
+
});
|
|
38
|
+
assert.deepStrictEqual(receivedMessage, {
|
|
39
|
+
responseId: "10",
|
|
40
|
+
data: tryAsU32(10),
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
it("should not let the sender consume its own pending message", () => {
|
|
44
|
+
const [txPort, rxPort] = DirectPort.pair();
|
|
45
|
+
txPort.postMessage("hello", codec.u32, {
|
|
46
|
+
responseId: "10",
|
|
47
|
+
data: tryAsU32(10),
|
|
48
|
+
});
|
|
49
|
+
let senderMessage = null;
|
|
50
|
+
txPort.on("hello", codec.u32, (msg) => {
|
|
51
|
+
senderMessage = msg;
|
|
52
|
+
});
|
|
53
|
+
assert.deepStrictEqual(senderMessage, null);
|
|
54
|
+
let receiverMessage = null;
|
|
55
|
+
rxPort.on("hello", codec.u32, (msg) => {
|
|
56
|
+
receiverMessage = msg;
|
|
57
|
+
});
|
|
58
|
+
assert.deepStrictEqual(receiverMessage, {
|
|
59
|
+
responseId: "10",
|
|
60
|
+
data: tryAsU32(10),
|
|
61
|
+
});
|
|
62
|
+
});
|
|
28
63
|
});
|
|
@@ -2,10 +2,20 @@ import type { MessagePort } from "node:worker_threads";
|
|
|
2
2
|
import { type Decode, type Encode } from "#@typeberry/codec";
|
|
3
3
|
import { ChainSpec } from "#@typeberry/config";
|
|
4
4
|
import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
|
|
5
|
+
import { FjallValuesSession } from "#@typeberry/database-fjall";
|
|
5
6
|
import { Blake2b } from "#@typeberry/hash";
|
|
6
7
|
import type { WorkerConfig } from "#@typeberry/workers-api";
|
|
7
8
|
import { ThreadPort, type TransferablePort } from "./port.js";
|
|
8
|
-
|
|
9
|
+
export { FjallValuesSession };
|
|
10
|
+
/** Persistent regular-node backend. */
|
|
11
|
+
export type PersistentBackend = "lmdb" | "fjall";
|
|
12
|
+
/** Transferable worker config for persistent regular-node workers. */
|
|
13
|
+
export type PersistentWorkerConfig<T> = LmdbWorkerConfig<T> | FjallWorkerConfig<T>;
|
|
14
|
+
/**
|
|
15
|
+
* Worker config for node.js, backed by the LMDB database.
|
|
16
|
+
*
|
|
17
|
+
* @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
|
|
18
|
+
*/
|
|
9
19
|
export declare class LmdbWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
|
|
10
20
|
readonly nodeName: string;
|
|
11
21
|
readonly chainSpec: ChainSpec;
|
|
@@ -28,18 +38,51 @@ export declare class LmdbWorkerConfig<T = void> implements WorkerConfig<T, Block
|
|
|
28
38
|
private constructor();
|
|
29
39
|
openDatabase(options?: {
|
|
30
40
|
readonly: boolean;
|
|
31
|
-
}): RootDb<BlocksDb, SerializedStatesDb
|
|
41
|
+
}): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
|
|
42
|
+
/** Convert this config into a thread-transferable object. */
|
|
43
|
+
intoTransferable(paramsCodec: Encode<T>): TransferableConfig;
|
|
44
|
+
}
|
|
45
|
+
/** Worker config for node.js, backed by a shared fjall engine. */
|
|
46
|
+
export declare class FjallWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
|
|
47
|
+
readonly nodeName: string;
|
|
48
|
+
readonly chainSpec: ChainSpec;
|
|
49
|
+
readonly workerParams: T;
|
|
50
|
+
readonly dbPath: string;
|
|
51
|
+
readonly blake2b: Blake2b;
|
|
52
|
+
readonly ports: Map<string, ThreadPort>;
|
|
53
|
+
readonly ephemeral: boolean;
|
|
54
|
+
readonly cacheSizeBytes: number | undefined;
|
|
55
|
+
static new<T>({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes, }: {
|
|
56
|
+
nodeName: string;
|
|
57
|
+
chainSpec: ChainSpec;
|
|
58
|
+
workerParams: T;
|
|
59
|
+
dbPath: string;
|
|
60
|
+
blake2b: Blake2b;
|
|
61
|
+
ports?: Map<string, ThreadPort>;
|
|
62
|
+
ephemeral?: boolean;
|
|
63
|
+
cacheSizeBytes?: number;
|
|
64
|
+
}): FjallWorkerConfig<T>;
|
|
65
|
+
/** Restore node config from a transferable config object. */
|
|
66
|
+
static fromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<FjallWorkerConfig<T>>;
|
|
67
|
+
private constructor();
|
|
68
|
+
openDatabase(options?: {
|
|
69
|
+
readonly: boolean;
|
|
70
|
+
}): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
|
|
32
71
|
/** Convert this config into a thread-transferable object. */
|
|
33
72
|
intoTransferable(paramsCodec: Encode<T>): TransferableConfig;
|
|
34
73
|
}
|
|
35
74
|
/** Config that's safe to transfer between worker threads. */
|
|
36
75
|
export type TransferableConfig = {
|
|
76
|
+
databaseBackend: PersistentBackend;
|
|
37
77
|
nodeName: string;
|
|
38
78
|
chainSpec: ChainSpec;
|
|
39
79
|
workerParams: Uint8Array;
|
|
40
80
|
dbPath: string;
|
|
41
81
|
workerPorts: [string, TransferablePort][];
|
|
82
|
+
cacheSizeBytes?: number;
|
|
42
83
|
};
|
|
84
|
+
/** Restore a persistent worker config from its transferable form. */
|
|
85
|
+
export declare function persistentConfigFromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<PersistentWorkerConfig<T>>;
|
|
43
86
|
/**
|
|
44
87
|
* Collect the transferable objects (communication ports) embedded in a config.
|
|
45
88
|
*
|
|
@@ -69,17 +112,16 @@ export declare class InMemWorkerConfig<T = undefined> implements WorkerConfig<T,
|
|
|
69
112
|
private constructor();
|
|
70
113
|
openDatabase(_options?: {
|
|
71
114
|
readonly: boolean;
|
|
72
|
-
}): RootDb<BlocksDb, SerializedStatesDb
|
|
115
|
+
}): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
|
|
73
116
|
}
|
|
74
117
|
/** Persistent values store backing the hybrid config. */
|
|
75
118
|
export type HybridBackend = "lmdb" | "fjall";
|
|
76
119
|
/**
|
|
77
120
|
* Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
|
|
78
|
-
* but large values persisted to disk
|
|
121
|
+
* but large values persisted to disk. The `backend` picks where the values go
|
|
122
|
+
* (lmdb or fjall).
|
|
79
123
|
*
|
|
80
|
-
*
|
|
81
|
-
* before committing to it. fjall opens its keyspace asynchronously, hence the
|
|
82
|
-
* async `new`.
|
|
124
|
+
* fjall opens its keyspace asynchronously, that is why `new` here is async.
|
|
83
125
|
*
|
|
84
126
|
* Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
|
|
85
127
|
* open/close/reopen dance that genesis init performs, so `openDatabase`
|
|
@@ -99,7 +141,7 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
|
|
|
99
141
|
readonly ephemeral: boolean;
|
|
100
142
|
readonly compression: boolean;
|
|
101
143
|
private readonly states;
|
|
102
|
-
static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, backend, }: {
|
|
144
|
+
static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, backend, sharedFjallSession, }: {
|
|
103
145
|
nodeName: string;
|
|
104
146
|
chainSpec: ChainSpec;
|
|
105
147
|
workerParams: T;
|
|
@@ -108,11 +150,18 @@ export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T
|
|
|
108
150
|
ephemeral?: boolean;
|
|
109
151
|
compression?: boolean;
|
|
110
152
|
backend?: HybridBackend;
|
|
153
|
+
/**
|
|
154
|
+
* Reuse an already-open fjall values session instead of opening a fresh
|
|
155
|
+
* keyspace. The fuzz target opens one per run and passes it on every reset,
|
|
156
|
+
* so only the in-memory blocks/leaf sets are rebuilt per vector. Ignored
|
|
157
|
+
* unless `backend === "fjall"`.
|
|
158
|
+
*/
|
|
159
|
+
sharedFjallSession?: FjallValuesSession;
|
|
111
160
|
}): Promise<HybridWorkerConfig<T>>;
|
|
112
161
|
private readonly blocks;
|
|
113
162
|
private constructor();
|
|
114
163
|
openDatabase(_options?: {
|
|
115
164
|
readonly: boolean;
|
|
116
|
-
}): RootDb<BlocksDb, SerializedStatesDb
|
|
165
|
+
}): Promise<RootDb<BlocksDb, SerializedStatesDb>>;
|
|
117
166
|
}
|
|
118
167
|
//# sourceMappingURL=config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAKL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAOnC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAI9D,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,uCAAuC;AACvC,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,OAAO,CAAC;AAEjD,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAEnF;;;;GAIG;AACH,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAI9B,SAAS,EAAE,OAAO;IA/CpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAgBpF,OAAO;IAaD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAahD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAU7D;AAED,kEAAkE;AAClE,qBAAa,iBAAiB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0C7E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAE9B,SAAS,EAAE,OAAO;aAClB,cAAc,EAAE,MAAM,GAAG,SAAS;IAjDpD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,EACjB,cAAc,GACf,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAiBpF,OAAO;IAYD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAuChD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAW7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,EAAE,iBAAiB,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAgBF,qEAAqE;AACrE,wBAAsB,gCAAgC,CAAC,CAAC,EACtD,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EACvB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAOpC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CAQjD;AAED,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;AAE7C;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0CnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;aAClB,WAAW,EAAE,OAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;WAhDZ,GAAG,CAAC,CAAC,EAAE,EAClB,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,EACjB,WAAkB,EAClB,OAAgB,EAChB,kBAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,OAAO,CAAC,EAAE,aAAa,CAAC;QACxB;;;;;WAKG;QACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KACzC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAYlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IAExC,OAAO;IAaD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CASjD"}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import { Decoder, Encoder } from "#@typeberry/codec";
|
|
2
2
|
import { ChainSpec } from "#@typeberry/config";
|
|
3
3
|
import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
|
|
4
|
-
import { HybridSerializedStates as FjallHybridSerializedStates } from "#@typeberry/database-fjall";
|
|
4
|
+
import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
|
|
5
5
|
import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
|
|
6
6
|
import { Blake2b } from "#@typeberry/hash";
|
|
7
7
|
import { ThreadPort } from "./port.js";
|
|
8
|
-
|
|
8
|
+
// Re-exported so the fuzz target can open one values session per run and reuse
|
|
9
|
+
// it across resets (see `HybridWorkerConfig` / `mainFuzz`).
|
|
10
|
+
export { FjallValuesSession };
|
|
11
|
+
/**
|
|
12
|
+
* Worker config for node.js, backed by the LMDB database.
|
|
13
|
+
*
|
|
14
|
+
* @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
|
|
15
|
+
*/
|
|
9
16
|
export class LmdbWorkerConfig {
|
|
10
17
|
nodeName;
|
|
11
18
|
chainSpec;
|
|
@@ -19,10 +26,10 @@ export class LmdbWorkerConfig {
|
|
|
19
26
|
}
|
|
20
27
|
/** Restore node config from a transferable config object. */
|
|
21
28
|
static async fromTransferable(decodeParams, config) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const
|
|
29
|
+
if (config.databaseBackend !== "lmdb") {
|
|
30
|
+
throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
|
|
31
|
+
}
|
|
32
|
+
const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
|
|
26
33
|
return LmdbWorkerConfig.new({
|
|
27
34
|
nodeName: config.nodeName,
|
|
28
35
|
chainSpec,
|
|
@@ -33,7 +40,7 @@ export class LmdbWorkerConfig {
|
|
|
33
40
|
});
|
|
34
41
|
}
|
|
35
42
|
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
36
|
-
// When set, the underlying
|
|
43
|
+
// When set, the underlying database skips fsync. Only safe for throwaway
|
|
37
44
|
// databases (the fuzz target wipes on reset). Not transferred to worker
|
|
38
45
|
// threads, so the durable main node path always gets the default.
|
|
39
46
|
ephemeral = false) {
|
|
@@ -45,7 +52,7 @@ export class LmdbWorkerConfig {
|
|
|
45
52
|
this.ports = ports;
|
|
46
53
|
this.ephemeral = ephemeral;
|
|
47
54
|
}
|
|
48
|
-
openDatabase(options = { readonly: true }) {
|
|
55
|
+
async openDatabase(options = { readonly: true }) {
|
|
49
56
|
const lmdb = LmdbRoot.new(this.dbPath, {
|
|
50
57
|
readOnly: options.readonly,
|
|
51
58
|
ephemeral: this.ephemeral,
|
|
@@ -59,14 +66,128 @@ export class LmdbWorkerConfig {
|
|
|
59
66
|
/** Convert this config into a thread-transferable object. */
|
|
60
67
|
intoTransferable(paramsCodec) {
|
|
61
68
|
return {
|
|
69
|
+
databaseBackend: "lmdb",
|
|
70
|
+
nodeName: this.nodeName,
|
|
71
|
+
chainSpec: this.chainSpec,
|
|
72
|
+
workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
|
|
73
|
+
dbPath: this.dbPath,
|
|
74
|
+
workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Worker config for node.js, backed by a shared fjall engine. */
|
|
79
|
+
export class FjallWorkerConfig {
|
|
80
|
+
nodeName;
|
|
81
|
+
chainSpec;
|
|
82
|
+
workerParams;
|
|
83
|
+
dbPath;
|
|
84
|
+
blake2b;
|
|
85
|
+
ports;
|
|
86
|
+
ephemeral;
|
|
87
|
+
cacheSizeBytes;
|
|
88
|
+
static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, }) {
|
|
89
|
+
return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes);
|
|
90
|
+
}
|
|
91
|
+
/** Restore node config from a transferable config object. */
|
|
92
|
+
static async fromTransferable(decodeParams, config) {
|
|
93
|
+
if (config.databaseBackend !== "fjall") {
|
|
94
|
+
throw new Error(`Expected fjall worker config, got ${config.databaseBackend}.`);
|
|
95
|
+
}
|
|
96
|
+
const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
|
|
97
|
+
return FjallWorkerConfig.new({
|
|
98
|
+
nodeName: config.nodeName,
|
|
99
|
+
chainSpec,
|
|
100
|
+
workerParams,
|
|
101
|
+
dbPath: config.dbPath,
|
|
102
|
+
blake2b,
|
|
103
|
+
ports,
|
|
104
|
+
cacheSizeBytes: config.cacheSizeBytes,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
108
|
+
// Kept for the fuzz/importer path. When set, persist() is skipped.
|
|
109
|
+
ephemeral = false, cacheSizeBytes = undefined) {
|
|
110
|
+
this.nodeName = nodeName;
|
|
111
|
+
this.chainSpec = chainSpec;
|
|
112
|
+
this.workerParams = workerParams;
|
|
113
|
+
this.dbPath = dbPath;
|
|
114
|
+
this.blake2b = blake2b;
|
|
115
|
+
this.ports = ports;
|
|
116
|
+
this.ephemeral = ephemeral;
|
|
117
|
+
this.cacheSizeBytes = cacheSizeBytes;
|
|
118
|
+
}
|
|
119
|
+
async openDatabase(options = { readonly: true }) {
|
|
120
|
+
const fjall = await FjallRoot.open(this.dbPath, {
|
|
121
|
+
readOnly: options.readonly,
|
|
122
|
+
ephemeral: this.ephemeral,
|
|
123
|
+
cacheSizeBytes: this.cacheSizeBytes,
|
|
124
|
+
});
|
|
125
|
+
let blocks = null;
|
|
126
|
+
let states = null;
|
|
127
|
+
try {
|
|
128
|
+
[blocks, states] = await Promise.all([
|
|
129
|
+
FjallBlocks.open(this.chainSpec, fjall),
|
|
130
|
+
FjallStates.open(this.chainSpec, this.blake2b, fjall),
|
|
131
|
+
]);
|
|
132
|
+
}
|
|
133
|
+
catch (e) {
|
|
134
|
+
await fjall.close();
|
|
135
|
+
throw e;
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
getBlocksDb: () => {
|
|
139
|
+
if (blocks === null) {
|
|
140
|
+
throw new Error("Fjall database is closed.");
|
|
141
|
+
}
|
|
142
|
+
return blocks;
|
|
143
|
+
},
|
|
144
|
+
getStatesDb: () => {
|
|
145
|
+
if (states === null) {
|
|
146
|
+
throw new Error("Fjall database is closed.");
|
|
147
|
+
}
|
|
148
|
+
return states;
|
|
149
|
+
},
|
|
150
|
+
close: async () => {
|
|
151
|
+
blocks = null;
|
|
152
|
+
states = null;
|
|
153
|
+
await fjall.close();
|
|
154
|
+
},
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
/** Convert this config into a thread-transferable object. */
|
|
158
|
+
intoTransferable(paramsCodec) {
|
|
159
|
+
return {
|
|
160
|
+
databaseBackend: "fjall",
|
|
62
161
|
nodeName: this.nodeName,
|
|
63
162
|
chainSpec: this.chainSpec,
|
|
64
163
|
workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
|
|
65
164
|
dbPath: this.dbPath,
|
|
66
165
|
workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
|
|
166
|
+
cacheSizeBytes: this.cacheSizeBytes,
|
|
67
167
|
};
|
|
68
168
|
}
|
|
69
169
|
}
|
|
170
|
+
async function decodeTransferableConfig(decodeParams, config) {
|
|
171
|
+
const blake2b = await Blake2b.createHasher();
|
|
172
|
+
const chainSpec = ChainSpec.new(config.chainSpec);
|
|
173
|
+
const workerParams = Decoder.decodeObject(decodeParams, config.workerParams, chainSpec);
|
|
174
|
+
const ports = new Map(config.workerPorts.map(([name, port]) => [name, ThreadPort.fromTransferable(chainSpec, port)]));
|
|
175
|
+
return {
|
|
176
|
+
blake2b,
|
|
177
|
+
chainSpec,
|
|
178
|
+
workerParams,
|
|
179
|
+
ports,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
/** Restore a persistent worker config from its transferable form. */
|
|
183
|
+
export async function persistentConfigFromTransferable(decodeParams, config) {
|
|
184
|
+
switch (config.databaseBackend) {
|
|
185
|
+
case "lmdb":
|
|
186
|
+
return LmdbWorkerConfig.fromTransferable(decodeParams, config);
|
|
187
|
+
case "fjall":
|
|
188
|
+
return FjallWorkerConfig.fromTransferable(decodeParams, config);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
70
191
|
/**
|
|
71
192
|
* Collect the transferable objects (communication ports) embedded in a config.
|
|
72
193
|
*
|
|
@@ -100,7 +221,7 @@ export class InMemWorkerConfig {
|
|
|
100
221
|
this.blocks = InMemoryBlocks.new();
|
|
101
222
|
this.states = InMemorySerializedStates.withHasher({ chainSpec, blake2b });
|
|
102
223
|
}
|
|
103
|
-
openDatabase(_options = { readonly: true }) {
|
|
224
|
+
async openDatabase(_options = { readonly: true }) {
|
|
104
225
|
// opening/closing db doesn't do anything, we persist the state.
|
|
105
226
|
return {
|
|
106
227
|
getBlocksDb: () => this.blocks,
|
|
@@ -111,11 +232,10 @@ export class InMemWorkerConfig {
|
|
|
111
232
|
}
|
|
112
233
|
/**
|
|
113
234
|
* Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
|
|
114
|
-
* but large values persisted to disk
|
|
235
|
+
* but large values persisted to disk. The `backend` picks where the values go
|
|
236
|
+
* (lmdb or fjall).
|
|
115
237
|
*
|
|
116
|
-
*
|
|
117
|
-
* before committing to it. fjall opens its keyspace asynchronously, hence the
|
|
118
|
-
* async `new`.
|
|
238
|
+
* fjall opens its keyspace asynchronously, that is why `new` here is async.
|
|
119
239
|
*
|
|
120
240
|
* Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
|
|
121
241
|
* open/close/reopen dance that genesis init performs, so `openDatabase`
|
|
@@ -135,11 +255,13 @@ export class HybridWorkerConfig {
|
|
|
135
255
|
ephemeral;
|
|
136
256
|
compression;
|
|
137
257
|
states;
|
|
138
|
-
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", }) {
|
|
139
|
-
//
|
|
140
|
-
//
|
|
258
|
+
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", sharedFjallSession, }) {
|
|
259
|
+
// The values store is created once here and shared across reopen. When a
|
|
260
|
+
// session is given (fuzz reset reuse) we wrap it instead of opening a new one.
|
|
141
261
|
const states = backend === "fjall"
|
|
142
|
-
?
|
|
262
|
+
? sharedFjallSession !== undefined
|
|
263
|
+
? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
|
|
264
|
+
: await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
|
|
143
265
|
: LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
|
|
144
266
|
return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
|
|
145
267
|
}
|
|
@@ -155,7 +277,7 @@ export class HybridWorkerConfig {
|
|
|
155
277
|
this.states = states;
|
|
156
278
|
this.blocks = InMemoryBlocks.new();
|
|
157
279
|
}
|
|
158
|
-
openDatabase(_options = { readonly: true }) {
|
|
280
|
+
async openDatabase(_options = { readonly: true }) {
|
|
159
281
|
return {
|
|
160
282
|
getBlocksDb: () => this.blocks,
|
|
161
283
|
getStatesDb: () => this.states,
|
|
@@ -2,11 +2,12 @@ import assert from "node:assert";
|
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
4
|
import { MessageChannel } from "node:worker_threads";
|
|
5
|
+
import { Bytes } from "#@typeberry/bytes";
|
|
5
6
|
import { codec } from "#@typeberry/codec";
|
|
6
7
|
import { tinyChainSpec } from "#@typeberry/config";
|
|
7
|
-
import { Blake2b } from "#@typeberry/hash";
|
|
8
|
+
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
8
9
|
import { tryAsU32 } from "#@typeberry/numbers";
|
|
9
|
-
import { configTransferList, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
|
|
10
|
+
import { configTransferList, FjallWorkerConfig, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
|
|
10
11
|
import { ThreadPort } from "./port.js";
|
|
11
12
|
const spec = tinyChainSpec;
|
|
12
13
|
describe("LmdbWorkerConfig transfer list", () => {
|
|
@@ -40,6 +41,54 @@ describe("LmdbWorkerConfig transfer list", () => {
|
|
|
40
41
|
}
|
|
41
42
|
});
|
|
42
43
|
});
|
|
44
|
+
describe("FjallWorkerConfig transfer list", () => {
|
|
45
|
+
it("surfaces embedded worker ports and marks the backend", async () => {
|
|
46
|
+
const blake2b = await Blake2b.createHasher();
|
|
47
|
+
const [portA, portB] = ThreadPort.pair(spec);
|
|
48
|
+
const config = FjallWorkerConfig.new({
|
|
49
|
+
nodeName: "node",
|
|
50
|
+
chainSpec: spec,
|
|
51
|
+
workerParams: tryAsU32(7),
|
|
52
|
+
dbPath: "db",
|
|
53
|
+
blake2b,
|
|
54
|
+
ports: new Map([["authorship-network", portA]]),
|
|
55
|
+
});
|
|
56
|
+
try {
|
|
57
|
+
const transferable = config.intoTransferable(codec.varU32);
|
|
58
|
+
assert.strictEqual(transferable.databaseBackend, "fjall");
|
|
59
|
+
assert.strictEqual(configTransferList(transferable).length, 1);
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
portA.close();
|
|
63
|
+
portB.close();
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
it("opens writable and read-only handles over one shared fjall path", async () => {
|
|
67
|
+
const blake2b = await Blake2b.createHasher();
|
|
68
|
+
const dbPath = fs.mkdtempSync("typeberry-fjall-worker-");
|
|
69
|
+
const config = FjallWorkerConfig.new({
|
|
70
|
+
nodeName: "node",
|
|
71
|
+
chainSpec: spec,
|
|
72
|
+
workerParams: undefined,
|
|
73
|
+
dbPath,
|
|
74
|
+
blake2b,
|
|
75
|
+
});
|
|
76
|
+
let writer = null;
|
|
77
|
+
let reader = null;
|
|
78
|
+
try {
|
|
79
|
+
writer = await config.openDatabase({ readonly: false });
|
|
80
|
+
reader = await config.openDatabase({ readonly: true });
|
|
81
|
+
const best = Bytes.fill(HASH_SIZE, 9).asOpaque();
|
|
82
|
+
await writer.getBlocksDb().setBestHeaderHash(best);
|
|
83
|
+
assert.strictEqual(reader.getBlocksDb().getBestHeaderHash().toString(), best.toString());
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await reader?.close();
|
|
87
|
+
await writer?.close();
|
|
88
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
});
|
|
43
92
|
describe("HybridWorkerConfig", () => {
|
|
44
93
|
// Both persistent backends must construct asynchronously and hand out a
|
|
45
94
|
// working db. fjall is the experimental backend we want to benchmark.
|
|
@@ -57,7 +106,7 @@ describe("HybridWorkerConfig", () => {
|
|
|
57
106
|
ephemeral: true,
|
|
58
107
|
backend,
|
|
59
108
|
});
|
|
60
|
-
const db = config.openDatabase({ readonly: false });
|
|
109
|
+
const db = await config.openDatabase({ readonly: false });
|
|
61
110
|
const states = db.getStatesDb();
|
|
62
111
|
try {
|
|
63
112
|
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
@@ -18,6 +18,7 @@ export declare class ThreadPort implements Port {
|
|
|
18
18
|
static pair(spec: ChainSpec): [ThreadPort, ThreadPort];
|
|
19
19
|
readonly threadId: number;
|
|
20
20
|
private readonly events;
|
|
21
|
+
private readonly pendingMessages;
|
|
21
22
|
static new(spec: ChainSpec, port: MessagePort): ThreadPort;
|
|
22
23
|
private constructor();
|
|
23
24
|
static fromTransferable(spec: ChainSpec, { port }: TransferablePort): ThreadPort;
|
|
@@ -28,5 +29,7 @@ export declare class ThreadPort implements Port {
|
|
|
28
29
|
on<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
|
|
29
30
|
once<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
|
|
30
31
|
postMessage<T>(event: string, codec: Codec<T>, msg: Envelope<T>): void;
|
|
32
|
+
private queuePending;
|
|
33
|
+
private flushPending;
|
|
31
34
|
}
|
|
32
35
|
//# sourceMappingURL=port.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAErE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAIF,qDAAqD;AACrD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,qBAAa,UAAW,YAAW,IAAI;IAgBnC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAhBvB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC;IAMtD,SAAgB,QAAQ,SAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAEhE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW;IAI7C,OAAO;IAmBP,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,gBAAgB;IAInE,gBAAgB,IAAI,gBAAgB;IAIpC,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI3C,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IAUvF,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IAoBzF,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;IAYtE,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,YAAY;CAUrB"}
|