@typeberry/lib 0.11.0-6310ba4 → 0.11.0-829b7ea
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -3
- package/packages/jam/config-node/node-config.d.ts +1 -11
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +3 -25
- package/packages/jam/config-node/node-config.test.js +3 -7
- package/packages/jam/database/states.d.ts +1 -8
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database/value-refs.d.ts +4 -5
- package/packages/jam/database/value-refs.d.ts.map +1 -1
- package/packages/jam/node/export.d.ts.map +1 -1
- package/packages/jam/node/export.js +8 -17
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +3 -5
- package/packages/jam/node/main-importer.d.ts +2 -6
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +12 -27
- package/packages/jam/node/main.d.ts.map +1 -1
- package/packages/jam/node/main.js +6 -16
- package/packages/jam/rpc/test/e2e-server.d.ts.map +1 -1
- package/packages/jam/rpc/test/e2e-server.js +3 -5
- package/packages/jam/rpc/test/e2e.js +1 -1
- package/packages/jam/state-merkleization/index.d.ts +1 -1
- package/packages/jam/state-merkleization/index.js +1 -1
- package/packages/workers/api-node/config.d.ts +6 -40
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +6 -82
- package/packages/workers/api-node/config.test.js +26 -63
- package/packages/jam/database-lmdb/blocks.d.ts +0 -24
- package/packages/jam/database-lmdb/blocks.d.ts.map +0 -1
- package/packages/jam/database-lmdb/blocks.js +0 -82
- package/packages/jam/database-lmdb/hybrid-states.d.ts +0 -56
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +0 -1
- package/packages/jam/database-lmdb/hybrid-states.js +0 -149
- package/packages/jam/database-lmdb/hybrid-states.test.d.ts +0 -2
- package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map +0 -1
- package/packages/jam/database-lmdb/hybrid-states.test.js +0 -180
- package/packages/jam/database-lmdb/index.d.ts +0 -5
- package/packages/jam/database-lmdb/index.d.ts.map +0 -1
- package/packages/jam/database-lmdb/index.js +0 -4
- package/packages/jam/database-lmdb/root.d.ts +0 -26
- package/packages/jam/database-lmdb/root.d.ts.map +0 -1
- package/packages/jam/database-lmdb/root.js +0 -53
- package/packages/jam/database-lmdb/states.d.ts +0 -76
- package/packages/jam/database-lmdb/states.d.ts.map +0 -1
- package/packages/jam/database-lmdb/states.js +0 -145
- package/packages/jam/database-lmdb/states.test.d.ts +0 -2
- package/packages/jam/database-lmdb/states.test.d.ts.map +0 -1
- package/packages/jam/database-lmdb/states.test.js +0 -202
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAKL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAKL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAI9D,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,uCAAuC;AACvC,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAExC,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAE7D,kEAAkE;AAClE,qBAAa,iBAAiB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aA0C7E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAE9B,SAAS,EAAE,OAAO;aAClB,cAAc,EAAE,MAAM,GAAG,SAAS;IAjDpD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,EACjB,cAAc,GACf,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAiBpF,OAAO;IAYD,YAAY,CAChB,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IAuChD,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAW7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,EAAE,iBAAiB,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAgBF,qEAAqE;AACrE,wBAAsB,gCAAgC,CAAC,CAAC,EACtD,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EACvB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAEpC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CAQjD;AAED,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC;AAEpC;;;;;;;;;;;;;;GAcG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAqCnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;aAClB,WAAW,EAAE,OAAO;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;WA3CZ,GAAG,CAAC,CAAC,EAAE,EAClB,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,EACjB,WAAkB,EAClB,kBAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB;;;;WAIG;QACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KACzC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAUlC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IAExC,OAAO;IAaD,YAAY,CAChB,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;CASjD"}
|
|
@@ -2,79 +2,11 @@ import { Decoder, Encoder } from "#@typeberry/codec";
|
|
|
2
2
|
import { ChainSpec } from "#@typeberry/config";
|
|
3
3
|
import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
|
|
4
4
|
import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
|
|
5
|
-
import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
|
|
6
5
|
import { Blake2b } from "#@typeberry/hash";
|
|
7
6
|
import { ThreadPort } from "./port.js";
|
|
8
7
|
// Re-exported so the fuzz target can open one values session per run and reuse
|
|
9
8
|
// it across resets (see `HybridWorkerConfig` / `mainFuzz`).
|
|
10
9
|
export { FjallValuesSession };
|
|
11
|
-
/**
|
|
12
|
-
* Worker config for node.js, backed by the LMDB database.
|
|
13
|
-
*
|
|
14
|
-
* @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
|
|
15
|
-
*/
|
|
16
|
-
export class LmdbWorkerConfig {
|
|
17
|
-
nodeName;
|
|
18
|
-
chainSpec;
|
|
19
|
-
workerParams;
|
|
20
|
-
dbPath;
|
|
21
|
-
blake2b;
|
|
22
|
-
ports;
|
|
23
|
-
ephemeral;
|
|
24
|
-
static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, }) {
|
|
25
|
-
return new LmdbWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral);
|
|
26
|
-
}
|
|
27
|
-
/** Restore node config from a transferable config object. */
|
|
28
|
-
static async fromTransferable(decodeParams, config) {
|
|
29
|
-
if (config.databaseBackend !== "lmdb") {
|
|
30
|
-
throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
|
|
31
|
-
}
|
|
32
|
-
const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
|
|
33
|
-
return LmdbWorkerConfig.new({
|
|
34
|
-
nodeName: config.nodeName,
|
|
35
|
-
chainSpec,
|
|
36
|
-
workerParams,
|
|
37
|
-
dbPath: config.dbPath,
|
|
38
|
-
blake2b,
|
|
39
|
-
ports,
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports,
|
|
43
|
-
// When set, the underlying database skips fsync. Only safe for throwaway
|
|
44
|
-
// databases (the fuzz target wipes on reset). Not transferred to worker
|
|
45
|
-
// threads, so the durable main node path always gets the default.
|
|
46
|
-
ephemeral = false) {
|
|
47
|
-
this.nodeName = nodeName;
|
|
48
|
-
this.chainSpec = chainSpec;
|
|
49
|
-
this.workerParams = workerParams;
|
|
50
|
-
this.dbPath = dbPath;
|
|
51
|
-
this.blake2b = blake2b;
|
|
52
|
-
this.ports = ports;
|
|
53
|
-
this.ephemeral = ephemeral;
|
|
54
|
-
}
|
|
55
|
-
async openDatabase(options = { readonly: true }) {
|
|
56
|
-
const lmdb = LmdbRoot.new(this.dbPath, {
|
|
57
|
-
readOnly: options.readonly,
|
|
58
|
-
ephemeral: this.ephemeral,
|
|
59
|
-
});
|
|
60
|
-
return {
|
|
61
|
-
getBlocksDb: () => LmdbBlocks.new(this.chainSpec, lmdb),
|
|
62
|
-
getStatesDb: () => LmdbStates.new(this.chainSpec, this.blake2b, lmdb),
|
|
63
|
-
close: async () => await lmdb.close(),
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
/** Convert this config into a thread-transferable object. */
|
|
67
|
-
intoTransferable(paramsCodec) {
|
|
68
|
-
return {
|
|
69
|
-
databaseBackend: "lmdb",
|
|
70
|
-
nodeName: this.nodeName,
|
|
71
|
-
chainSpec: this.chainSpec,
|
|
72
|
-
workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
|
|
73
|
-
dbPath: this.dbPath,
|
|
74
|
-
workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
10
|
/** Worker config for node.js, backed by a shared fjall engine. */
|
|
79
11
|
export class FjallWorkerConfig {
|
|
80
12
|
nodeName;
|
|
@@ -181,12 +113,7 @@ async function decodeTransferableConfig(decodeParams, config) {
|
|
|
181
113
|
}
|
|
182
114
|
/** Restore a persistent worker config from its transferable form. */
|
|
183
115
|
export async function persistentConfigFromTransferable(decodeParams, config) {
|
|
184
|
-
|
|
185
|
-
case "lmdb":
|
|
186
|
-
return LmdbWorkerConfig.fromTransferable(decodeParams, config);
|
|
187
|
-
case "fjall":
|
|
188
|
-
return FjallWorkerConfig.fromTransferable(decodeParams, config);
|
|
189
|
-
}
|
|
116
|
+
return FjallWorkerConfig.fromTransferable(decodeParams, config);
|
|
190
117
|
}
|
|
191
118
|
/**
|
|
192
119
|
* Collect the transferable objects (communication ports) embedded in a config.
|
|
@@ -232,8 +159,7 @@ export class InMemWorkerConfig {
|
|
|
232
159
|
}
|
|
233
160
|
/**
|
|
234
161
|
* Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
|
|
235
|
-
* but large values persisted to disk
|
|
236
|
-
* (lmdb or fjall).
|
|
162
|
+
* but large values persisted to disk via fjall.
|
|
237
163
|
*
|
|
238
164
|
* fjall opens its keyspace asynchronously, that is why `new` here is async.
|
|
239
165
|
*
|
|
@@ -255,14 +181,12 @@ export class HybridWorkerConfig {
|
|
|
255
181
|
ephemeral;
|
|
256
182
|
compression;
|
|
257
183
|
states;
|
|
258
|
-
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true,
|
|
184
|
+
static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, sharedFjallSession, }) {
|
|
259
185
|
// The values store is created once here and shared across reopen. When a
|
|
260
186
|
// session is given (fuzz reset reuse) we wrap it instead of opening a new one.
|
|
261
|
-
const states =
|
|
262
|
-
?
|
|
263
|
-
|
|
264
|
-
: await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
|
|
265
|
-
: LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
|
|
187
|
+
const states = sharedFjallSession !== undefined
|
|
188
|
+
? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
|
|
189
|
+
: await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral });
|
|
266
190
|
return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
|
|
267
191
|
}
|
|
268
192
|
blocks;
|
|
@@ -1,46 +1,14 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
|
-
import { MessageChannel } from "node:worker_threads";
|
|
5
4
|
import { Bytes } from "#@typeberry/bytes";
|
|
6
5
|
import { codec } from "#@typeberry/codec";
|
|
7
6
|
import { tinyChainSpec } from "#@typeberry/config";
|
|
8
7
|
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
9
8
|
import { tryAsU32 } from "#@typeberry/numbers";
|
|
10
|
-
import { configTransferList, FjallWorkerConfig, HybridWorkerConfig
|
|
9
|
+
import { configTransferList, FjallWorkerConfig, HybridWorkerConfig } from "./config.js";
|
|
11
10
|
import { ThreadPort } from "./port.js";
|
|
12
11
|
const spec = tinyChainSpec;
|
|
13
|
-
describe("LmdbWorkerConfig transfer list", () => {
|
|
14
|
-
it("surfaces embedded worker ports so they can be transferred", async () => {
|
|
15
|
-
const blake2b = await Blake2b.createHasher();
|
|
16
|
-
const [portA, portB] = ThreadPort.pair(spec);
|
|
17
|
-
const config = LmdbWorkerConfig.new({
|
|
18
|
-
nodeName: "node",
|
|
19
|
-
chainSpec: spec,
|
|
20
|
-
workerParams: tryAsU32(7),
|
|
21
|
-
dbPath: "db",
|
|
22
|
-
blake2b,
|
|
23
|
-
ports: new Map([["authorship-network", portA]]),
|
|
24
|
-
});
|
|
25
|
-
const transferable = config.intoTransferable(codec.varU32);
|
|
26
|
-
const transferList = configTransferList(transferable);
|
|
27
|
-
// the single embedded comms port must be reported for transfer
|
|
28
|
-
assert.strictEqual(transferList.length, 1);
|
|
29
|
-
const sink = new MessageChannel();
|
|
30
|
-
try {
|
|
31
|
-
// reproduces the bug: a config carrying a port cannot be cloned without
|
|
32
|
-
// listing that port in the transfer list.
|
|
33
|
-
assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
|
|
34
|
-
// with the ports surfaced, posting succeeds.
|
|
35
|
-
assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
|
|
36
|
-
}
|
|
37
|
-
finally {
|
|
38
|
-
sink.port1.close();
|
|
39
|
-
sink.port2.close();
|
|
40
|
-
portB.close();
|
|
41
|
-
}
|
|
42
|
-
});
|
|
43
|
-
});
|
|
44
12
|
describe("FjallWorkerConfig transfer list", () => {
|
|
45
13
|
it("surfaces embedded worker ports and marks the backend", async () => {
|
|
46
14
|
const blake2b = await Blake2b.createHasher();
|
|
@@ -90,38 +58,33 @@ describe("FjallWorkerConfig transfer list", () => {
|
|
|
90
58
|
});
|
|
91
59
|
});
|
|
92
60
|
describe("HybridWorkerConfig", () => {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
61
|
+
it("constructs and opens a fjall-backed hybrid db", async () => {
|
|
62
|
+
const blake2b = await Blake2b.createHasher();
|
|
63
|
+
const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-");
|
|
64
|
+
try {
|
|
65
|
+
const config = await HybridWorkerConfig.new({
|
|
66
|
+
nodeName: "node",
|
|
67
|
+
chainSpec: spec,
|
|
68
|
+
workerParams: undefined,
|
|
69
|
+
blake2b,
|
|
70
|
+
dbPath,
|
|
71
|
+
ephemeral: true,
|
|
72
|
+
});
|
|
73
|
+
const db = await config.openDatabase({ readonly: false });
|
|
74
|
+
const states = db.getStatesDb();
|
|
99
75
|
try {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
chainSpec: spec,
|
|
103
|
-
workerParams: undefined,
|
|
104
|
-
blake2b,
|
|
105
|
-
dbPath,
|
|
106
|
-
ephemeral: true,
|
|
107
|
-
backend,
|
|
108
|
-
});
|
|
109
|
-
const db = await config.openDatabase({ readonly: false });
|
|
110
|
-
const states = db.getStatesDb();
|
|
111
|
-
try {
|
|
112
|
-
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
113
|
-
assert.notStrictEqual(states, undefined);
|
|
114
|
-
}
|
|
115
|
-
finally {
|
|
116
|
-
// The values store owns the on-disk resources (the no-op db.close()
|
|
117
|
-
// does not), so close it explicitly to release the fjall keyspace.
|
|
118
|
-
await states.close();
|
|
119
|
-
await db.close();
|
|
120
|
-
}
|
|
76
|
+
assert.notStrictEqual(db.getBlocksDb(), undefined);
|
|
77
|
+
assert.notStrictEqual(states, undefined);
|
|
121
78
|
}
|
|
122
79
|
finally {
|
|
123
|
-
|
|
80
|
+
// The values store owns the on-disk resources (the no-op db.close()
|
|
81
|
+
// does not), so close it explicitly to release the fjall keyspace.
|
|
82
|
+
await states.close();
|
|
83
|
+
await db.close();
|
|
124
84
|
}
|
|
125
|
-
}
|
|
126
|
-
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
fs.rmSync(dbPath, { recursive: true, force: true });
|
|
88
|
+
}
|
|
89
|
+
});
|
|
127
90
|
});
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { type BlockView, type ExtrinsicView, type HeaderHash, type HeaderView, type StateRootHash } from "#@typeberry/block";
|
|
2
|
-
import type { ChainSpec } from "#@typeberry/config";
|
|
3
|
-
import type { BlocksDb } from "#@typeberry/database/blocks.js";
|
|
4
|
-
import { type WithHash } from "#@typeberry/hash";
|
|
5
|
-
import type { LmdbRoot } from "./root.js";
|
|
6
|
-
export declare class LmdbBlocks implements BlocksDb {
|
|
7
|
-
private readonly chainSpec;
|
|
8
|
-
private readonly root;
|
|
9
|
-
private readonly extrinsics;
|
|
10
|
-
private readonly headers;
|
|
11
|
-
private readonly postStateRoots;
|
|
12
|
-
static new(chainSpec: ChainSpec, root: LmdbRoot): LmdbBlocks;
|
|
13
|
-
private constructor();
|
|
14
|
-
setPostStateRoot(hash: HeaderHash, postStateRoot: StateRootHash): Promise<void>;
|
|
15
|
-
getPostStateRoot(hash: HeaderHash): StateRootHash | null;
|
|
16
|
-
insertBlock(block: WithHash<HeaderHash, BlockView>): Promise<void>;
|
|
17
|
-
setBestHeaderHash(hash: HeaderHash): Promise<void>;
|
|
18
|
-
getBestHeaderHash(): HeaderHash;
|
|
19
|
-
getHeader(hash: HeaderHash): HeaderView | null;
|
|
20
|
-
getExtrinsic(hash: HeaderHash): ExtrinsicView | null;
|
|
21
|
-
markUnused(hash: HeaderHash): void;
|
|
22
|
-
close(): Promise<void>;
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=blocks.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAQjD,qBAAa,UAAW,YAAW,QAAQ;IAUvC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAEvC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ;IAI/C,OAAO;IAeD,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrF,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQlD,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IASlE,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxD,iBAAiB,IAAI,UAAU;IAS/B,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAS9C,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQpD,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAM5B,KAAK;CAGZ"}
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { Extrinsic, Header, } from "#@typeberry/block";
|
|
2
|
-
import { Bytes } from "#@typeberry/bytes";
|
|
3
|
-
import { Decoder } from "#@typeberry/codec";
|
|
4
|
-
import { HASH_SIZE } from "#@typeberry/hash";
|
|
5
|
-
const BEST_BLOCK = "best hash and posterior state root";
|
|
6
|
-
// TODO [ToDr] consider having a changeset for transactions,
|
|
7
|
-
// where we store all `insert ++ key ++ value` and `remove ++ key`
|
|
8
|
-
// in a single `Uint8Array` JAM-encoded. That could then
|
|
9
|
-
// be efficiently transferred between threads.
|
|
10
|
-
export class LmdbBlocks {
|
|
11
|
-
chainSpec;
|
|
12
|
-
root;
|
|
13
|
-
extrinsics;
|
|
14
|
-
headers;
|
|
15
|
-
postStateRoots;
|
|
16
|
-
static new(chainSpec, root) {
|
|
17
|
-
return new LmdbBlocks(chainSpec, root);
|
|
18
|
-
}
|
|
19
|
-
constructor(chainSpec, root) {
|
|
20
|
-
this.chainSpec = chainSpec;
|
|
21
|
-
this.root = root;
|
|
22
|
-
// NOTE [ToDr] Extrinsics are stored under header hash, not their hash. Revise if it's an issue.
|
|
23
|
-
this.extrinsics = this.root.subDb("extrinsics");
|
|
24
|
-
this.headers = this.root.subDb("headers");
|
|
25
|
-
// NOTE [ToDr] We currently store all posterior state roots, however it's
|
|
26
|
-
// most likely very redundant. We probably only need to store the posterior
|
|
27
|
-
// state roots of recent blocks to be able to quickly resolve forks
|
|
28
|
-
// OR we need a way to be able to traverse the blocks history forward
|
|
29
|
-
// (i.e. know what next block(s) is).
|
|
30
|
-
this.postStateRoots = this.root.subDb("postStateRoots");
|
|
31
|
-
}
|
|
32
|
-
async setPostStateRoot(hash, postStateRoot) {
|
|
33
|
-
await this.postStateRoots.put(hash.raw, postStateRoot.raw);
|
|
34
|
-
}
|
|
35
|
-
getPostStateRoot(hash) {
|
|
36
|
-
const postStateRoot = this.postStateRoots.get(hash.raw);
|
|
37
|
-
if (postStateRoot === undefined) {
|
|
38
|
-
return null;
|
|
39
|
-
}
|
|
40
|
-
return Bytes.fromBlob(postStateRoot, HASH_SIZE).asOpaque();
|
|
41
|
-
}
|
|
42
|
-
async insertBlock(block) {
|
|
43
|
-
const header = block.data.header.view().encoded();
|
|
44
|
-
const extrinsic = block.data.extrinsic.view().encoded();
|
|
45
|
-
await this.root.db.transaction(() => {
|
|
46
|
-
this.headers.put(block.hash.raw, header.raw);
|
|
47
|
-
this.extrinsics.put(block.hash.raw, extrinsic.raw);
|
|
48
|
-
});
|
|
49
|
-
}
|
|
50
|
-
async setBestHeaderHash(hash) {
|
|
51
|
-
await this.root.db.put(BEST_BLOCK, hash.raw);
|
|
52
|
-
}
|
|
53
|
-
getBestHeaderHash() {
|
|
54
|
-
const bestHeaderHash = this.root.db.get(BEST_BLOCK);
|
|
55
|
-
if (bestHeaderHash === undefined) {
|
|
56
|
-
return Bytes.zero(HASH_SIZE).asOpaque();
|
|
57
|
-
}
|
|
58
|
-
return Bytes.fromBlob(bestHeaderHash, HASH_SIZE).asOpaque();
|
|
59
|
-
}
|
|
60
|
-
getHeader(hash) {
|
|
61
|
-
const data = this.headers.get(hash.raw);
|
|
62
|
-
if (data === undefined) {
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
return Decoder.decodeObject(Header.Codec.View, data, this.chainSpec);
|
|
66
|
-
}
|
|
67
|
-
getExtrinsic(hash) {
|
|
68
|
-
const data = this.extrinsics.get(hash.raw);
|
|
69
|
-
if (data === undefined) {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
return Decoder.decodeObject(Extrinsic.Codec.View, data, this.chainSpec);
|
|
73
|
-
}
|
|
74
|
-
markUnused(hash) {
|
|
75
|
-
this.headers.removeSync(hash.raw);
|
|
76
|
-
this.extrinsics.removeSync(hash.raw);
|
|
77
|
-
this.postStateRoots.removeSync(hash.raw);
|
|
78
|
-
}
|
|
79
|
-
async close() {
|
|
80
|
-
await Promise.all([this.headers.close(), this.extrinsics.close(), this.postStateRoots.close()]);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import type { HeaderHash, StateRootHash } from "#@typeberry/block";
|
|
2
|
-
import type { ChainSpec } from "#@typeberry/config";
|
|
3
|
-
import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
|
|
4
|
-
import type { Blake2b } from "#@typeberry/hash";
|
|
5
|
-
import type { ServicesUpdate, State } from "#@typeberry/state";
|
|
6
|
-
import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
|
|
7
|
-
import { OK, Result } from "#@typeberry/utils";
|
|
8
|
-
/**
|
|
9
|
-
* Hybrid serialized-states db.
|
|
10
|
-
*
|
|
11
|
-
* States (leafs) are kept in-memory, but large values are persisted to lmdb.
|
|
12
|
-
* Reads go straight to lmdb, which keeps its own page cache.
|
|
13
|
-
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
14
|
-
* keep the heap usage bounded.
|
|
15
|
-
*
|
|
16
|
-
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
17
|
-
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
18
|
-
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
19
|
-
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
20
|
-
* run are never collected.
|
|
21
|
-
*/
|
|
22
|
-
export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
|
|
23
|
-
private readonly spec;
|
|
24
|
-
private readonly blake2b;
|
|
25
|
-
private readonly root;
|
|
26
|
-
private readonly inMemStates;
|
|
27
|
-
private readonly lmdbValues;
|
|
28
|
-
private readonly valuesDb;
|
|
29
|
-
private readonly refsStore;
|
|
30
|
-
private readonly refs;
|
|
31
|
-
private pendingCleanup;
|
|
32
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
|
|
33
|
-
spec: ChainSpec;
|
|
34
|
-
blake2b: Blake2b;
|
|
35
|
-
dbPath: string;
|
|
36
|
-
readOnly?: boolean;
|
|
37
|
-
ephemeral?: boolean;
|
|
38
|
-
compression?: boolean;
|
|
39
|
-
}): HybridSerializedStates;
|
|
40
|
-
private constructor();
|
|
41
|
-
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
42
|
-
updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
|
|
43
|
-
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
44
|
-
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
45
|
-
commitFinalized(headers: HeaderHash[]): void;
|
|
46
|
-
markUnused(header: HeaderHash): void;
|
|
47
|
-
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
48
|
-
private applyRefs;
|
|
49
|
-
diskSizeInBytes(): number | null;
|
|
50
|
-
close(): Promise<void>;
|
|
51
|
-
/** Write new large values to LMDB in one transaction. */
|
|
52
|
-
private writeValues;
|
|
53
|
-
/** Read a value from LMDB. */
|
|
54
|
-
private readValue;
|
|
55
|
-
}
|
|
56
|
-
//# sourceMappingURL=hybrid-states.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -1,149 +0,0 @@
|
|
|
1
|
-
import { HashDictionary, SortedSet } from "#@typeberry/collections";
|
|
2
|
-
import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
|
|
3
|
-
import { Logger } from "#@typeberry/logger";
|
|
4
|
-
import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
|
|
5
|
-
import { leafComparator } from "#@typeberry/trie";
|
|
6
|
-
import { OK, Result } from "#@typeberry/utils";
|
|
7
|
-
import { LmdbRoot } from "./root.js";
|
|
8
|
-
const logger = Logger.new(import.meta.filename, "db");
|
|
9
|
-
/**
|
|
10
|
-
* Hybrid serialized-states db.
|
|
11
|
-
*
|
|
12
|
-
* States (leafs) are kept in-memory, but large values are persisted to lmdb.
|
|
13
|
-
* Reads go straight to lmdb, which keeps its own page cache.
|
|
14
|
-
* NOTE: this DB is designed for long fuzzing and to be used with pruning to
|
|
15
|
-
* keep the heap usage bounded.
|
|
16
|
-
*
|
|
17
|
-
* Values that no longer belong to any surviving state are removed from lmdb,
|
|
18
|
-
* decided by in-memory refcounting (`ValueRefs`) driven by the importer's
|
|
19
|
-
* finality signal. Counts are not persisted: this db cannot resume from disk
|
|
20
|
-
* anyway (the leaf sets live in memory), so values left over by a previous
|
|
21
|
-
* run are never collected.
|
|
22
|
-
*/
|
|
23
|
-
export class HybridSerializedStates {
|
|
24
|
-
spec;
|
|
25
|
-
blake2b;
|
|
26
|
-
root;
|
|
27
|
-
inMemStates = HashDictionary.new();
|
|
28
|
-
lmdbValues;
|
|
29
|
-
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
30
|
-
valuesDb;
|
|
31
|
-
refsStore = new InMemoryValueRefsStore();
|
|
32
|
-
refs = new ValueRefs(this.refsStore);
|
|
33
|
-
// Queue of not-yet-committed value removals, awaited on close.
|
|
34
|
-
pendingCleanup = Promise.resolve();
|
|
35
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
|
|
36
|
-
const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
|
|
37
|
-
return new HybridSerializedStates(spec, blake2b, root);
|
|
38
|
-
}
|
|
39
|
-
constructor(spec, blake2b, root) {
|
|
40
|
-
this.spec = spec;
|
|
41
|
-
this.blake2b = blake2b;
|
|
42
|
-
this.root = root;
|
|
43
|
-
this.lmdbValues = this.root.subDb("values");
|
|
44
|
-
this.valuesDb = { get: (key) => this.readValue(key) };
|
|
45
|
-
}
|
|
46
|
-
async insertInitialState(headerHash, entries) {
|
|
47
|
-
const { values, leafs } = updateLeafs(SortedSet.fromArray(leafComparator, []), this.blake2b, Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
|
|
48
|
-
const res = await this.writeValues(values);
|
|
49
|
-
if (res.isError) {
|
|
50
|
-
return res;
|
|
51
|
-
}
|
|
52
|
-
this.inMemStates.set(headerHash, leafs);
|
|
53
|
-
this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
|
|
54
|
-
return Result.ok(OK);
|
|
55
|
-
}
|
|
56
|
-
async updateAndSetState(header, state, update) {
|
|
57
|
-
const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
|
|
58
|
-
// Clone the leaf set before mutating: the previous state keeps using its own.
|
|
59
|
-
const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
|
|
60
|
-
const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
61
|
-
const res = await this.writeValues(values);
|
|
62
|
-
if (res.isError) {
|
|
63
|
-
// Leave the caller's state untouched: its new leaves would reference
|
|
64
|
-
// values that never reached disk.
|
|
65
|
-
return res;
|
|
66
|
-
}
|
|
67
|
-
// Re-create the lookup with the shared values accessor only once the new
|
|
68
|
-
// values are durably written.
|
|
69
|
-
state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
|
|
70
|
-
this.inMemStates.set(header, leafs);
|
|
71
|
-
this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
|
|
72
|
-
return Result.ok(OK);
|
|
73
|
-
}
|
|
74
|
-
async getStateRoot(state) {
|
|
75
|
-
return state.backend.getStateRoot(this.blake2b);
|
|
76
|
-
}
|
|
77
|
-
getState(header) {
|
|
78
|
-
const leafs = this.inMemStates.get(header);
|
|
79
|
-
if (leafs === undefined) {
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
|
|
83
|
-
return SerializedState.new(this.spec, this.blake2b, leafDb);
|
|
84
|
-
}
|
|
85
|
-
commitFinalized(headers) {
|
|
86
|
-
this.applyRefs(this.refs.commitFinalized(headers));
|
|
87
|
-
}
|
|
88
|
-
markUnused(header) {
|
|
89
|
-
// Release the speculative references first (a no-op for finalized states,
|
|
90
|
-
// whose deltas were already consumed by `commitFinalized`).
|
|
91
|
-
this.applyRefs(this.refs.releaseUnfinalized(header));
|
|
92
|
-
this.inMemStates.delete(header);
|
|
93
|
-
}
|
|
94
|
-
/** Apply a refcounting update and remove values that lost their last reference. */
|
|
95
|
-
applyRefs(update) {
|
|
96
|
-
this.refsStore.apply(update);
|
|
97
|
-
if (update.removeValues.length === 0) {
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
// Queued, not awaited: a failed removal only leaks a value.
|
|
101
|
-
this.pendingCleanup = this.pendingCleanup
|
|
102
|
-
.then(() => this.lmdbValues.transaction(() => {
|
|
103
|
-
for (const v of update.removeValues) {
|
|
104
|
-
this.lmdbValues.remove(v.raw);
|
|
105
|
-
}
|
|
106
|
-
}))
|
|
107
|
-
.catch((e) => {
|
|
108
|
-
logger.error `Failed to remove unreferenced values: ${e}`;
|
|
109
|
-
});
|
|
110
|
-
}
|
|
111
|
-
diskSizeInBytes() {
|
|
112
|
-
return this.root.sizeInBytes();
|
|
113
|
-
}
|
|
114
|
-
async close() {
|
|
115
|
-
await this.pendingCleanup;
|
|
116
|
-
await this.lmdbValues.close();
|
|
117
|
-
await this.root.close();
|
|
118
|
-
}
|
|
119
|
-
/** Write new large values to LMDB in one transaction. */
|
|
120
|
-
async writeValues(values) {
|
|
121
|
-
if (values.length === 0) {
|
|
122
|
-
return Result.ok(OK);
|
|
123
|
-
}
|
|
124
|
-
try {
|
|
125
|
-
// Flush queued removals first: a pending cleanup might be about to delete
|
|
126
|
-
// a content hash we are re-inserting now. Writing behind it keeps the
|
|
127
|
-
// re-inserted value on disk (removals are only queued at refcount 0, so
|
|
128
|
-
// none can be queued for a value still referenced).
|
|
129
|
-
await this.pendingCleanup;
|
|
130
|
-
await this.lmdbValues.transaction(() => {
|
|
131
|
-
for (const [hash, val] of values) {
|
|
132
|
-
this.lmdbValues.put(hash.raw, val.raw);
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
|
-
}
|
|
136
|
-
catch (e) {
|
|
137
|
-
return Result.error(StateUpdateError.Commit, () => `Failed to commit values: ${e}`);
|
|
138
|
-
}
|
|
139
|
-
return Result.ok(OK);
|
|
140
|
-
}
|
|
141
|
-
/** Read a value from LMDB. */
|
|
142
|
-
readValue(key) {
|
|
143
|
-
const val = this.lmdbValues.get(key.raw);
|
|
144
|
-
if (val === undefined) {
|
|
145
|
-
throw new Error(`Missing value at key: ${key}`);
|
|
146
|
-
}
|
|
147
|
-
return val;
|
|
148
|
-
}
|
|
149
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.test.ts"],"names":[],"mappings":""}
|