@typeberry/lib 0.8.4-9587022 → 0.8.4-faebc7a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -4
- package/packages/jam/database/states.d.ts +7 -0
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.d.ts +45 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.js +113 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts +2 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.test.js +83 -0
- package/packages/jam/database-fjall/index.d.ts +3 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -0
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +52 -0
- package/packages/jam/database-fjall/root.d.ts.map +1 -0
- package/packages/jam/database-fjall/root.js +85 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +3 -1
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/hybrid-states.js +5 -2
- package/packages/jam/database-lmdb/root.d.ts +14 -1
- package/packages/jam/database-lmdb/root.d.ts.map +1 -1
- package/packages/jam/database-lmdb/root.js +25 -5
- package/packages/jam/database-lmdb/states.d.ts +1 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/states.js +3 -0
- package/packages/jam/database-lmdb/states.test.js +4 -4
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts +18 -10
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map +1 -1
- package/packages/jam/jamnp-s/tasks/ticket-distribution.js +44 -68
- package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js +30 -8
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +21 -4
- package/packages/jam/node/main-importer.d.ts +7 -4
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +10 -4
- package/packages/jam/safrole/bandersnatch-vrf.d.ts +2 -2
- package/packages/jam/safrole/bandersnatch-vrf.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-vrf.js +9 -6
- package/packages/jam/safrole/bandersnatch-vrf.test.js +9 -7
- package/packages/jam/safrole/safrole.js +5 -5
- package/packages/jam/safrole/safrole.test.js +13 -13
- package/packages/jam/ticket-pool/index.d.ts +4 -0
- package/packages/jam/ticket-pool/index.d.ts.map +1 -0
- package/packages/jam/ticket-pool/index.js +3 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts +30 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.js +56 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.js +67 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts +46 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.js +29 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts +2 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.test.js +34 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts +24 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.js +37 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.js +54 -0
- package/packages/workers/api-node/config.d.ts +14 -5
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +29 -20
- package/packages/workers/api-node/config.test.js +38 -1
- package/packages/workers/block-authorship/main.d.ts.map +1 -1
- package/packages/workers/block-authorship/main.js +22 -72
- package/packages/workers/block-authorship/ticket-validator.d.ts +32 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-validator.js +56 -0
- package/packages/workers/comms-authorship-network/protocol.d.ts +10 -0
- package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/protocol.js +8 -1
- package/packages/workers/importer/importer.d.ts +1 -1
- package/packages/workers/importer/importer.d.ts.map +1 -1
- package/packages/workers/importer/importer.js +2 -2
- package/packages/workers/importer/{events-logger.d.ts → stats.d.ts} +7 -2
- package/packages/workers/importer/stats.d.ts.map +1 -0
- package/packages/workers/importer/{events-logger.js → stats.js} +21 -2
- package/packages/workers/jam-network/main.d.ts.map +1 -1
- package/packages/workers/jam-network/main.js +20 -4
- package/packages/workers/importer/events-logger.d.ts.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@typeberry/lib",
|
|
3
|
-
"version": "0.8.4-
|
|
3
|
+
"version": "0.8.4-faebc7a",
|
|
4
4
|
"description": "Typeberry Library",
|
|
5
5
|
"main": "./bin/lib/index.js",
|
|
6
6
|
"types": "./bin/lib/index.d.ts",
|
|
@@ -154,8 +154,6 @@
|
|
|
154
154
|
"#@typeberry/pvm/*": "./bin/pvm/*",
|
|
155
155
|
"#@typeberry/rpc": "./bin/rpc/index.js",
|
|
156
156
|
"#@typeberry/rpc/*": "./bin/rpc/*",
|
|
157
|
-
"#@typeberry/tci": "./bin/tci/index.js",
|
|
158
|
-
"#@typeberry/tci/*": "./bin/tci/*",
|
|
159
157
|
"#@typeberry/test-runner": "./bin/test-runner/index.js",
|
|
160
158
|
"#@typeberry/test-runner/*": "./bin/test-runner/*",
|
|
161
159
|
"#@typeberry/configs": "./packages/configs/index.js",
|
|
@@ -216,6 +214,8 @@
|
|
|
216
214
|
"#@typeberry/config-node/*": "./packages/jam/config-node/*",
|
|
217
215
|
"#@typeberry/database": "./packages/jam/database/index.js",
|
|
218
216
|
"#@typeberry/database/*": "./packages/jam/database/*",
|
|
217
|
+
"#@typeberry/database-fjall": "./packages/jam/database-fjall/index.js",
|
|
218
|
+
"#@typeberry/database-fjall/*": "./packages/jam/database-fjall/*",
|
|
219
219
|
"#@typeberry/database-lmdb": "./packages/jam/database-lmdb/index.js",
|
|
220
220
|
"#@typeberry/database-lmdb/*": "./packages/jam/database-lmdb/*",
|
|
221
221
|
"#@typeberry/executor": "./packages/jam/executor/index.js",
|
|
@@ -244,6 +244,8 @@
|
|
|
244
244
|
"#@typeberry/state-merkleization/*": "./packages/jam/state-merkleization/*",
|
|
245
245
|
"#@typeberry/state-vectors": "./packages/jam/state-vectors/index.js",
|
|
246
246
|
"#@typeberry/state-vectors/*": "./packages/jam/state-vectors/*",
|
|
247
|
+
"#@typeberry/ticket-pool": "./packages/jam/ticket-pool/index.js",
|
|
248
|
+
"#@typeberry/ticket-pool/*": "./packages/jam/ticket-pool/*",
|
|
247
249
|
"#@typeberry/transition": "./packages/jam/transition/index.js",
|
|
248
250
|
"#@typeberry/transition/*": "./packages/jam/transition/*",
|
|
249
251
|
"#@typeberry/disputes": "./packages/jam/transition/disputes/index.js",
|
|
@@ -267,7 +269,7 @@
|
|
|
267
269
|
"@fluffylabs/anan-as": "^1.4.0",
|
|
268
270
|
"@noble/ed25519": "2.2.3",
|
|
269
271
|
"hash-wasm": "4.12.0",
|
|
270
|
-
"@typeberry/native": "0.
|
|
272
|
+
"@typeberry/native": "0.5.1",
|
|
271
273
|
"eventemitter3": "^5.0.1",
|
|
272
274
|
"@opentelemetry/api": "1.9.0"
|
|
273
275
|
},
|
|
@@ -35,6 +35,13 @@ export interface StatesDb<T extends State = State> {
|
|
|
35
35
|
getState(header: HeaderHash): T | null;
|
|
36
36
|
/** Mark state as no longer needed. Backend may remove it asynchronously. */
|
|
37
37
|
markUnused(header: HeaderHash): void;
|
|
38
|
+
/**
|
|
39
|
+
* Apparent on-disk size of the database in bytes, used for monitoring.
|
|
40
|
+
*
|
|
41
|
+
* Returns `null` when the size is unknown. Backends that are not persisted
|
|
42
|
+
* (e.g. in-memory) may omit the method entirely.
|
|
43
|
+
*/
|
|
44
|
+
diskSizeInBytes?(): number | null;
|
|
38
45
|
/** Close the database and free resources. */
|
|
39
46
|
close(): Promise<void>;
|
|
40
47
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { HeaderHash, StateRootHash } from "#@typeberry/block";
|
|
2
|
+
import type { ChainSpec } from "#@typeberry/config";
|
|
3
|
+
import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
|
|
4
|
+
import type { Blake2b } from "#@typeberry/hash";
|
|
5
|
+
import type { ServicesUpdate, State } from "#@typeberry/state";
|
|
6
|
+
import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
|
|
7
|
+
import { OK, Result } from "#@typeberry/utils";
|
|
8
|
+
/**
|
|
9
|
+
* Hybrid serialized-states db (fjall variant).
|
|
10
|
+
*
|
|
11
|
+
* States (leafs) are kept in-memory, but large values are persisted to fjall.
|
|
12
|
+
* Reads go straight to fjall, which keeps its own (bounded) block cache.
|
|
13
|
+
* Designed for long fuzzing, used with pruning to keep heap usage bounded.
|
|
14
|
+
*
|
|
15
|
+
* Behaviourally identical to the LMDB hybrid db; differences are mechanical:
|
|
16
|
+
* construction is async, and value writes are ordered + flushed explicitly
|
|
17
|
+
* because fjall has no transaction primitive.
|
|
18
|
+
*/
|
|
19
|
+
export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
|
|
20
|
+
private readonly spec;
|
|
21
|
+
private readonly blake2b;
|
|
22
|
+
private readonly root;
|
|
23
|
+
private readonly values;
|
|
24
|
+
static new({ spec, blake2b, dbPath, ephemeral, }: {
|
|
25
|
+
spec: ChainSpec;
|
|
26
|
+
blake2b: Blake2b;
|
|
27
|
+
dbPath: string;
|
|
28
|
+
ephemeral?: boolean;
|
|
29
|
+
}): Promise<HybridSerializedStates>;
|
|
30
|
+
private readonly inMemStates;
|
|
31
|
+
private readonly valuesDb;
|
|
32
|
+
private constructor();
|
|
33
|
+
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
34
|
+
updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
35
|
+
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
36
|
+
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
37
|
+
markUnused(header: HeaderHash): void;
|
|
38
|
+
diskSizeInBytes(): number | null;
|
|
39
|
+
close(): Promise<void>;
|
|
40
|
+
/** Write new large values to fjall, then flush. */
|
|
41
|
+
private writeValues;
|
|
42
|
+
/** Read a value from fjall. */
|
|
43
|
+
private readValue;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=hybrid-states.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;GAUG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAsBxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;WAxBZ,GAAG,CAAC,EACf,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,GACV,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IAErG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAIX,mDAAmD;YACrC,WAAW;IAgBzB,+BAA+B;IAC/B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { HashDictionary, SortedSet } from "#@typeberry/collections";
|
|
2
|
+
import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
|
|
3
|
+
import { Logger } from "#@typeberry/logger";
|
|
4
|
+
import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
|
|
5
|
+
import { leafComparator } from "#@typeberry/trie";
|
|
6
|
+
import { OK, Result } from "#@typeberry/utils";
|
|
7
|
+
import { FjallRoot, toUint8Array } from "./root.js";
|
|
8
|
+
const logger = Logger.new(import.meta.filename, "db");
|
|
9
|
+
/**
|
|
10
|
+
* Hybrid serialized-states db (fjall variant).
|
|
11
|
+
*
|
|
12
|
+
* States (leafs) are kept in-memory, but large values are persisted to fjall.
|
|
13
|
+
* Reads go straight to fjall, which keeps its own (bounded) block cache.
|
|
14
|
+
* Designed for long fuzzing, used with pruning to keep heap usage bounded.
|
|
15
|
+
*
|
|
16
|
+
* Behaviourally identical to the LMDB hybrid db; differences are mechanical:
|
|
17
|
+
* construction is async, and value writes are ordered + flushed explicitly
|
|
18
|
+
* because fjall has no transaction primitive.
|
|
19
|
+
*/
|
|
20
|
+
export class HybridSerializedStates {
|
|
21
|
+
spec;
|
|
22
|
+
blake2b;
|
|
23
|
+
root;
|
|
24
|
+
values;
|
|
25
|
+
static async new({ spec, blake2b, dbPath, ephemeral, }) {
|
|
26
|
+
const root = await FjallRoot.open(dbPath, { ephemeral });
|
|
27
|
+
const values = await root.partition("values");
|
|
28
|
+
return new HybridSerializedStates(spec, blake2b, root, values);
|
|
29
|
+
}
|
|
30
|
+
inMemStates = HashDictionary.new();
|
|
31
|
+
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
32
|
+
valuesDb;
|
|
33
|
+
constructor(spec, blake2b, root, values) {
|
|
34
|
+
this.spec = spec;
|
|
35
|
+
this.blake2b = blake2b;
|
|
36
|
+
this.root = root;
|
|
37
|
+
this.values = values;
|
|
38
|
+
this.valuesDb = { get: (key) => this.readValue(key) };
|
|
39
|
+
}
|
|
40
|
+
async insertInitialState(headerHash, entries) {
|
|
41
|
+
const { values, leafs } = updateLeafs(SortedSet.fromArray(leafComparator, []), this.blake2b, Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
|
|
42
|
+
const res = await this.writeValues(values);
|
|
43
|
+
if (res.isError) {
|
|
44
|
+
return res;
|
|
45
|
+
}
|
|
46
|
+
this.inMemStates.set(headerHash, leafs);
|
|
47
|
+
return Result.ok(OK);
|
|
48
|
+
}
|
|
49
|
+
async updateAndSetState(header, state, update) {
|
|
50
|
+
const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
|
|
51
|
+
// Clone the leaf set before mutating: the previous state keeps using its own.
|
|
52
|
+
const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
|
|
53
|
+
const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
54
|
+
const res = await this.writeValues(values);
|
|
55
|
+
if (res.isError) {
|
|
56
|
+
// Leave the caller's state untouched: its new leaves would reference
|
|
57
|
+
// values that never reached disk.
|
|
58
|
+
return res;
|
|
59
|
+
}
|
|
60
|
+
// Re-create the lookup with the shared values accessor only once the new
|
|
61
|
+
// values are durably written.
|
|
62
|
+
state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
|
|
63
|
+
this.inMemStates.set(header, leafs);
|
|
64
|
+
return Result.ok(OK);
|
|
65
|
+
}
|
|
66
|
+
async getStateRoot(state) {
|
|
67
|
+
return state.backend.getStateRoot(this.blake2b);
|
|
68
|
+
}
|
|
69
|
+
getState(header) {
|
|
70
|
+
const leafs = this.inMemStates.get(header);
|
|
71
|
+
if (leafs === undefined) {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
|
|
75
|
+
return SerializedState.new(this.spec, this.blake2b, leafDb);
|
|
76
|
+
}
|
|
77
|
+
markUnused(header) {
|
|
78
|
+
// We only remove the state from memory - values are not pruned at all,
|
|
79
|
+
// but since they are stored on disk we should be safe.
|
|
80
|
+
this.inMemStates.delete(header);
|
|
81
|
+
}
|
|
82
|
+
diskSizeInBytes() {
|
|
83
|
+
return this.root.sizeInBytes();
|
|
84
|
+
}
|
|
85
|
+
async close() {
|
|
86
|
+
await this.root.close();
|
|
87
|
+
}
|
|
88
|
+
/** Write new large values to fjall, then flush. */
|
|
89
|
+
async writeValues(values) {
|
|
90
|
+
if (values.length === 0) {
|
|
91
|
+
return Result.ok(OK);
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
for (const [hash, val] of values) {
|
|
95
|
+
await this.values.insert(hash.raw, val.raw);
|
|
96
|
+
}
|
|
97
|
+
await this.root.persist();
|
|
98
|
+
}
|
|
99
|
+
catch (e) {
|
|
100
|
+
logger.error `${e}`;
|
|
101
|
+
return Result.error(StateUpdateError.Commit, () => `Failed to commit values: ${e}`);
|
|
102
|
+
}
|
|
103
|
+
return Result.ok(OK);
|
|
104
|
+
}
|
|
105
|
+
/** Read a value from fjall. */
|
|
106
|
+
readValue(key) {
|
|
107
|
+
const val = toUint8Array(this.values.get(key.raw));
|
|
108
|
+
if (val === null) {
|
|
109
|
+
throw new Error(`Missing value at key: ${key}`);
|
|
110
|
+
}
|
|
111
|
+
return val;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid-states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { afterEach, before, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import { Bytes, BytesBlob } from "#@typeberry/bytes";
|
|
5
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
6
|
+
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
7
|
+
import { InMemoryState } from "#@typeberry/state";
|
|
8
|
+
import { StateEntries } from "#@typeberry/state-merkleization";
|
|
9
|
+
import { deepEqual, OK, Result } from "#@typeberry/utils";
|
|
10
|
+
import { HybridSerializedStates } from "./hybrid-states.js";
|
|
11
|
+
let blake2b;
|
|
12
|
+
before(async () => {
|
|
13
|
+
blake2b = await Blake2b.createHasher();
|
|
14
|
+
});
|
|
15
|
+
function createTempDir(suffix = "fjall-hybrid") {
|
|
16
|
+
return fs.mkdtempSync(`typeberry-${suffix}`);
|
|
17
|
+
}
|
|
18
|
+
describe("Fjall hybrid serialized states", () => {
|
|
19
|
+
const spec = tinyChainSpec;
|
|
20
|
+
const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
21
|
+
let dbPath = "";
|
|
22
|
+
beforeEach(() => {
|
|
23
|
+
dbPath = createTempDir();
|
|
24
|
+
});
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
fs.rmSync(dbPath, { recursive: true });
|
|
27
|
+
});
|
|
28
|
+
it("round-trips an initial state through the on-disk values store", async () => {
|
|
29
|
+
const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
|
|
30
|
+
try {
|
|
31
|
+
const empty = InMemoryState.empty(spec);
|
|
32
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
|
|
33
|
+
const expectedRoot = serialized.getRootHash(blake2b);
|
|
34
|
+
const res = await states.insertInitialState(headerHash, serialized);
|
|
35
|
+
deepEqual(res, Result.ok(OK));
|
|
36
|
+
const state = states.getState(headerHash);
|
|
37
|
+
assert.ok(state !== null);
|
|
38
|
+
const stateRoot = await states.getStateRoot(state);
|
|
39
|
+
assert.strictEqual(`${stateRoot}`, `${expectedRoot}`);
|
|
40
|
+
deepEqual(InMemoryState.copyFrom(spec, state, new Map()), empty);
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await states.close();
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
it("reads large values back from disk", async () => {
|
|
47
|
+
const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
|
|
48
|
+
try {
|
|
49
|
+
// > 32 bytes => stored in the values db (not embedded in the leaf).
|
|
50
|
+
const big1 = BytesBlob.blobFromString("x".repeat(100));
|
|
51
|
+
const big2 = BytesBlob.blobFromString("y".repeat(100));
|
|
52
|
+
const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
|
|
53
|
+
const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
54
|
+
const entries = StateEntries.fromEntriesUnsafe([
|
|
55
|
+
[key1, big1],
|
|
56
|
+
[key2, big2],
|
|
57
|
+
]);
|
|
58
|
+
const res = await states.insertInitialState(headerHash, entries);
|
|
59
|
+
deepEqual(res, Result.ok(OK));
|
|
60
|
+
const state = states.getState(headerHash);
|
|
61
|
+
assert.ok(state !== null);
|
|
62
|
+
assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
|
|
63
|
+
assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
await states.close();
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
it("drops the leaf set on markUnused while values stay on disk", async () => {
|
|
70
|
+
const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
|
|
71
|
+
try {
|
|
72
|
+
const empty = InMemoryState.empty(spec);
|
|
73
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
|
|
74
|
+
await states.insertInitialState(headerHash, serialized);
|
|
75
|
+
assert.ok(states.getState(headerHash) !== null);
|
|
76
|
+
states.markUnused(headerHash);
|
|
77
|
+
assert.strictEqual(states.getState(headerHash), null);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await states.close();
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { type Partition } from "@fjall-js/fjall";
|
|
2
|
+
export type { Partition };
|
|
3
|
+
/**
|
|
4
|
+
* Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
|
|
5
|
+
*/
|
|
6
|
+
export declare function toUint8Array(value: Buffer | null): Uint8Array | null;
|
|
7
|
+
export type FjallRootOptions = {
|
|
8
|
+
/**
|
|
9
|
+
* When set, durability flushes (`persist`) are skipped entirely.
|
|
10
|
+
*
|
|
11
|
+
* Only safe for throwaway databases (e.g. the fuzz target, which wipes on
|
|
12
|
+
* every reset). Mirrors LMDB's `noSync`.
|
|
13
|
+
*/
|
|
14
|
+
ephemeral?: boolean;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* A thin abstraction over the fjall keyspace.
|
|
18
|
+
*
|
|
19
|
+
* Unlike LMDB (a memory-mapped B-tree), fjall is an LSM-tree that reads/writes
|
|
20
|
+
* through regular file I/O, so its working set is bounded by an explicit block
|
|
21
|
+
* cache rather than the whole mmap. LMDB has been causing oom issues because
|
|
22
|
+
* of that.
|
|
23
|
+
*/
|
|
24
|
+
export declare class FjallRoot {
|
|
25
|
+
private readonly keyspace;
|
|
26
|
+
/** Path of the underlying keyspace directory, used to report on-disk usage. */
|
|
27
|
+
private readonly dbPath;
|
|
28
|
+
private readonly options;
|
|
29
|
+
private constructor();
|
|
30
|
+
/** Open (or create) a fjall keyspace at the given path. */
|
|
31
|
+
static open(dbPath: string, options: FjallRootOptions): Promise<FjallRoot>;
|
|
32
|
+
/** Open (or create) a partition under this keyspace. */
|
|
33
|
+
partition(name: string): Promise<Partition>;
|
|
34
|
+
/**
|
|
35
|
+
* Flush the journal to disk so prior writes survive a crash.
|
|
36
|
+
*
|
|
37
|
+
* Call after a logically-complete unit of work (one block, one state commit).
|
|
38
|
+
* A no-op for ephemeral databases.
|
|
39
|
+
*/
|
|
40
|
+
persist(): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* Apparent on-disk size of the keyspace directory, in bytes.
|
|
43
|
+
*
|
|
44
|
+
* Returns `null` if the directory cannot be walked (e.g. not yet created).
|
|
45
|
+
* Unlike LMDB's single `data.mdb`, a fjall keyspace is a directory of
|
|
46
|
+
* partition and journal files, so we sum it recursively.
|
|
47
|
+
*/
|
|
48
|
+
sizeInBytes(): number | null;
|
|
49
|
+
/** Persist with `sync-all` and release the keyspace handle. */
|
|
50
|
+
close(): Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=root.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEtE,YAAY,EAAE,SAAS,EAAE,CAAC;AAE1B;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAKpE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,SAAS;IAElB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO;IAOP,2DAA2D;WAC9C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC;IAKhF,wDAAwD;IAClD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAIjD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;;;;;OAMG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { open } from "@fjall-js/fjall";
|
|
4
|
+
/**
|
|
5
|
+
* Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
|
|
6
|
+
*/
|
|
7
|
+
export function toUint8Array(value) {
|
|
8
|
+
if (value === null) {
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A thin abstraction over the fjall keyspace.
|
|
15
|
+
*
|
|
16
|
+
* Unlike LMDB (a memory-mapped B-tree), fjall is an LSM-tree that reads/writes
|
|
17
|
+
* through regular file I/O, so its working set is bounded by an explicit block
|
|
18
|
+
* cache rather than the whole mmap. LMDB has been causing oom issues because
|
|
19
|
+
* of that.
|
|
20
|
+
*/
|
|
21
|
+
export class FjallRoot {
|
|
22
|
+
keyspace;
|
|
23
|
+
dbPath;
|
|
24
|
+
options;
|
|
25
|
+
constructor(keyspace,
|
|
26
|
+
/** Path of the underlying keyspace directory, used to report on-disk usage. */
|
|
27
|
+
dbPath, options) {
|
|
28
|
+
this.keyspace = keyspace;
|
|
29
|
+
this.dbPath = dbPath;
|
|
30
|
+
this.options = options;
|
|
31
|
+
}
|
|
32
|
+
/** Open (or create) a fjall keyspace at the given path. */
|
|
33
|
+
static async open(dbPath, options) {
|
|
34
|
+
const keyspace = await open(dbPath);
|
|
35
|
+
return new FjallRoot(keyspace, dbPath, options);
|
|
36
|
+
}
|
|
37
|
+
/** Open (or create) a partition under this keyspace. */
|
|
38
|
+
async partition(name) {
|
|
39
|
+
return this.keyspace.partition(name);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Flush the journal to disk so prior writes survive a crash.
|
|
43
|
+
*
|
|
44
|
+
* Call after a logically-complete unit of work (one block, one state commit).
|
|
45
|
+
* A no-op for ephemeral databases.
|
|
46
|
+
*/
|
|
47
|
+
async persist() {
|
|
48
|
+
if (this.options.ephemeral === true) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
await this.keyspace.persist();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Apparent on-disk size of the keyspace directory, in bytes.
|
|
55
|
+
*
|
|
56
|
+
* Returns `null` if the directory cannot be walked (e.g. not yet created).
|
|
57
|
+
* Unlike LMDB's single `data.mdb`, a fjall keyspace is a directory of
|
|
58
|
+
* partition and journal files, so we sum it recursively.
|
|
59
|
+
*/
|
|
60
|
+
sizeInBytes() {
|
|
61
|
+
try {
|
|
62
|
+
return dirSizeInBytes(this.dbPath);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Persist with `sync-all` and release the keyspace handle. */
|
|
69
|
+
async close() {
|
|
70
|
+
await this.keyspace.close();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function dirSizeInBytes(dir) {
|
|
74
|
+
let total = 0;
|
|
75
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
76
|
+
const full = path.join(dir, entry.name);
|
|
77
|
+
if (entry.isDirectory()) {
|
|
78
|
+
total += dirSizeInBytes(full);
|
|
79
|
+
}
|
|
80
|
+
else if (entry.isFile()) {
|
|
81
|
+
total += fs.statSync(full).size;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return total;
|
|
85
|
+
}
|
|
@@ -20,12 +20,13 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
20
20
|
private readonly inMemStates;
|
|
21
21
|
private readonly lmdbValues;
|
|
22
22
|
private readonly valuesDb;
|
|
23
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, }: {
|
|
23
|
+
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
|
|
24
24
|
spec: ChainSpec;
|
|
25
25
|
blake2b: Blake2b;
|
|
26
26
|
dbPath: string;
|
|
27
27
|
readOnly?: boolean;
|
|
28
28
|
ephemeral?: boolean;
|
|
29
|
+
compression?: boolean;
|
|
29
30
|
}): HybridSerializedStates;
|
|
30
31
|
private constructor();
|
|
31
32
|
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
@@ -33,6 +34,7 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
33
34
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
34
35
|
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
35
36
|
markUnused(header: HeaderHash): void;
|
|
37
|
+
diskSizeInBytes(): number | null;
|
|
36
38
|
close(): Promise<void>;
|
|
37
39
|
/** Write new large values to LMDB in one transaction. */
|
|
38
40
|
private writeValues;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -20,8 +20,8 @@ export class HybridSerializedStates {
|
|
|
20
20
|
lmdbValues;
|
|
21
21
|
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
22
22
|
valuesDb;
|
|
23
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, }) {
|
|
24
|
-
const root = LmdbRoot.new(dbPath, readOnly, ephemeral);
|
|
23
|
+
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
|
|
24
|
+
const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
|
|
25
25
|
return new HybridSerializedStates(spec, blake2b, root);
|
|
26
26
|
}
|
|
27
27
|
constructor(spec, blake2b, root) {
|
|
@@ -73,6 +73,9 @@ export class HybridSerializedStates {
|
|
|
73
73
|
// but since they are stored on disk we should be safe.
|
|
74
74
|
this.inMemStates.delete(header);
|
|
75
75
|
}
|
|
76
|
+
diskSizeInBytes() {
|
|
77
|
+
return this.root.sizeInBytes();
|
|
78
|
+
}
|
|
76
79
|
async close() {
|
|
77
80
|
await this.lmdbValues.close();
|
|
78
81
|
await this.root.close();
|
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import * as lmdb from "lmdb";
|
|
2
2
|
export type SubDb = lmdb.Database<Uint8Array, lmdb.Key>;
|
|
3
|
+
export type LmdbRootOptions = {
|
|
4
|
+
readOnly?: boolean;
|
|
5
|
+
ephemeral?: boolean;
|
|
6
|
+
compression?: boolean;
|
|
7
|
+
};
|
|
3
8
|
/** A thin abstraction over lmdb database interface. */
|
|
4
9
|
export declare class LmdbRoot {
|
|
5
10
|
readonly db: lmdb.RootDatabase<Uint8Array, lmdb.Key>;
|
|
6
|
-
|
|
11
|
+
/** Path of the underlying LMDB data file, used to report on-disk usage. */
|
|
12
|
+
private readonly dataFilePath;
|
|
13
|
+
static new(dbPath: string, options: LmdbRootOptions): LmdbRoot;
|
|
7
14
|
private constructor();
|
|
8
15
|
/** Open a sub-database under the same path. */
|
|
9
16
|
subDb(name: string): SubDb;
|
|
17
|
+
/**
|
|
18
|
+
* Apparent on-disk size of the LMDB data file, in bytes.
|
|
19
|
+
*
|
|
20
|
+
* Returns `null` if the file cannot be `stat`-ed (e.g. not yet created).
|
|
21
|
+
*/
|
|
22
|
+
sizeInBytes(): number | null;
|
|
10
23
|
/** Close the database and all sub-databases. */
|
|
11
24
|
close(): Promise<void>;
|
|
12
25
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/root.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/root.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;AAExD,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,uDAAuD;AACvD,qBAAa,QAAQ;IACnB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACrD,2EAA2E;IAC3E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe;IAInD,OAAO;IAwBP,+CAA+C;IAC/C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;IAI1B;;;;OAIG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,gDAAgD;IAC1C,KAAK;CAGZ"}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
1
2
|
import * as lmdb from "lmdb";
|
|
2
3
|
/** A thin abstraction over lmdb database interface. */
|
|
3
4
|
export class LmdbRoot {
|
|
4
5
|
db;
|
|
5
|
-
|
|
6
|
-
|
|
6
|
+
/** Path of the underlying LMDB data file, used to report on-disk usage. */
|
|
7
|
+
dataFilePath;
|
|
8
|
+
static new(dbPath, options) {
|
|
9
|
+
return new LmdbRoot(dbPath, options);
|
|
7
10
|
}
|
|
8
|
-
constructor(dbPath, readOnly
|
|
11
|
+
constructor(dbPath, { readOnly, ephemeral, compression }) {
|
|
12
|
+
// `lmdb.open` treats an extension-less path as a directory and stores the
|
|
13
|
+
// environment in `<dbPath>/data.mdb` (next to `lock.mdb`).
|
|
14
|
+
this.dataFilePath = `${dbPath}/data.mdb`;
|
|
15
|
+
const isEphemeral = ephemeral ?? false;
|
|
9
16
|
this.db = lmdb.open(dbPath, {
|
|
10
17
|
// experimental options
|
|
11
18
|
noMemInit: true,
|
|
@@ -15,8 +22,8 @@ export class LmdbRoot {
|
|
|
15
22
|
// durability is pointless, so we skip fsync and skip compressing the large
|
|
16
23
|
// per-block leaf blobs. Both are pure overhead there and dominate the cost.
|
|
17
24
|
// This trades disk space (uncompressed) and crash-durability for speed.
|
|
18
|
-
compression: !
|
|
19
|
-
noSync:
|
|
25
|
+
compression: compression ?? !isEphemeral,
|
|
26
|
+
noSync: isEphemeral,
|
|
20
27
|
keyEncoding: "binary",
|
|
21
28
|
encoding: "binary",
|
|
22
29
|
readOnly,
|
|
@@ -26,6 +33,19 @@ export class LmdbRoot {
|
|
|
26
33
|
subDb(name) {
|
|
27
34
|
return this.db.openDB({ name });
|
|
28
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Apparent on-disk size of the LMDB data file, in bytes.
|
|
38
|
+
*
|
|
39
|
+
* Returns `null` if the file cannot be `stat`-ed (e.g. not yet created).
|
|
40
|
+
*/
|
|
41
|
+
sizeInBytes() {
|
|
42
|
+
try {
|
|
43
|
+
return fs.statSync(this.dataFilePath).size;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
29
49
|
/** Close the database and all sub-databases. */
|
|
30
50
|
async close() {
|
|
31
51
|
await this.db.close();
|
|
@@ -69,6 +69,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
|
|
|
69
69
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
70
70
|
getState(root: HeaderHash): SerializedState<LeafDb> | null;
|
|
71
71
|
markUnused(header: HeaderHash): void;
|
|
72
|
+
diskSizeInBytes(): number | null;
|
|
72
73
|
close(): Promise<void>;
|
|
73
74
|
}
|
|
74
75
|
//# sourceMappingURL=states.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
|