@typeberry/lib 0.10.0-5a5e4ab → 0.10.0-89fc6fa
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +27 -13
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +1 -1
- package/packages/workers/importer/main.d.ts +2 -0
- package/packages/workers/importer/main.d.ts.map +1 -1
- package/packages/workers/importer/main.js +1 -1
package/package.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAsBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6LrH"}
|
|
@@ -25,6 +25,8 @@ const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
|
|
|
25
25
|
* the resident memory bounded.
|
|
26
26
|
*/
|
|
27
27
|
const FUZZ_FJALL_CACHE_BYTES = 128 * 1024 * 1024;
|
|
28
|
+
/** Rebuild the fjall-hybrid values session every N resets to limit LSM read amplification. */
|
|
29
|
+
const REBUILD_FJALL_SESSION_EVERY = 50;
|
|
28
30
|
/**
|
|
29
31
|
* Resolve the directory the fuzzer should use for its on-disk database, or
|
|
30
32
|
* `undefined` for an in-memory database. The dedicated `FUZZ_DB_SUBDIR` is
|
|
@@ -75,6 +77,8 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
75
77
|
// every reset, because opening it is the slow part. Only the in-memory blocks
|
|
76
78
|
// and leaf sets are rebuilt for each vector. fjall-hybrid only.
|
|
77
79
|
let fjallSession = null;
|
|
80
|
+
// Track how many times resetState has been called for periodic fjall session rebuilds.
|
|
81
|
+
let resetCount = 0;
|
|
78
82
|
// Set when close() starts. Guards resetState so a fuzz command arriving
|
|
79
83
|
// mid-shutdown can't build a fresh node that close() then orphans.
|
|
80
84
|
let isClosing = false;
|
|
@@ -112,6 +116,8 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
112
116
|
runningNode = null;
|
|
113
117
|
await finish;
|
|
114
118
|
}
|
|
119
|
+
// Increment reset counter for periodic fjall session rebuilds.
|
|
120
|
+
resetCount++;
|
|
115
121
|
const buildNode = (databaseBasePath) => {
|
|
116
122
|
const isPersistent = databaseBasePath !== undefined;
|
|
117
123
|
return mainImporter({
|
|
@@ -141,21 +147,16 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
141
147
|
stateBackend: isPersistent ? hybridStateBackend : "lmdb",
|
|
142
148
|
// Reuse the session keyspace (fjall-hybrid only, other backends
|
|
143
149
|
// ignore it). Nothing to pass for the in-memory fallback.
|
|
144
|
-
sharedFjallSession:
|
|
150
|
+
sharedFjallSession: hybridStateBackend === "fjall-hybrid" ? (fjallSession ?? undefined) : undefined,
|
|
145
151
|
});
|
|
146
152
|
};
|
|
147
153
|
if (fuzzDbBase !== undefined) {
|
|
148
154
|
try {
|
|
149
155
|
if (hybridStateBackend === FUZZ_DB_FJALL) {
|
|
150
|
-
// fjall-hybrid:
|
|
151
|
-
//
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
// already initialized from the in-memory blocks, which we rebuild on
|
|
155
|
-
// every reset, not from the values store, so reusing it does not
|
|
156
|
-
// resume the previous run.
|
|
157
|
-
if (fjallSession === null) {
|
|
158
|
-
// Start from a clean slate once, then keep the keyspace open.
|
|
156
|
+
// fjall-hybrid: manage a reused values session.
|
|
157
|
+
// Rebuild it periodically to avoid LSM read amplification.
|
|
158
|
+
if (resetCount === 1) {
|
|
159
|
+
// First reset: start from a clean slate.
|
|
159
160
|
await wipeFuzzDb(fuzzDbBase);
|
|
160
161
|
fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
|
|
161
162
|
ephemeral: true,
|
|
@@ -163,11 +164,24 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
163
164
|
});
|
|
164
165
|
logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
|
|
165
166
|
}
|
|
167
|
+
if (resetCount % REBUILD_FJALL_SESSION_EVERY === 0 && fjallSession !== null) {
|
|
168
|
+
// Periodic rebuild: close, wipe session dir, and reopen.
|
|
169
|
+
const session = fjallSession;
|
|
170
|
+
fjallSession = null;
|
|
171
|
+
await session.close().catch(() => { });
|
|
172
|
+
await wipeFuzzDb(`${fuzzDbBase}/${FUZZ_FJALL_VALUES_SUBDIR}`).catch(() => { });
|
|
173
|
+
}
|
|
174
|
+
if (fjallSession === null) {
|
|
175
|
+
// No active session: create a fresh one.
|
|
176
|
+
fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
|
|
177
|
+
ephemeral: true,
|
|
178
|
+
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
179
|
+
});
|
|
180
|
+
logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
|
|
181
|
+
}
|
|
166
182
|
}
|
|
167
183
|
else {
|
|
168
|
-
//
|
|
169
|
-
// reset. A fresh db each reset makes `initializeDatabase` set up
|
|
170
|
-
// genesis again instead of resuming the previous run.
|
|
184
|
+
// All other backends ("fjall", "lmdb", "lmdb-hybrid"): wipe and reopen on every reset.
|
|
171
185
|
await wipeFuzzDb(fuzzDbBase);
|
|
172
186
|
}
|
|
173
187
|
runningNode = await buildNode(fuzzDbBase);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAoHlB"}
|
|
@@ -79,9 +79,9 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
79
79
|
await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry, {
|
|
80
80
|
initGenesisFromAncestry: options.initGenesisFromAncestry,
|
|
81
81
|
});
|
|
82
|
-
await rootDb.close();
|
|
83
82
|
const { db, importer } = await createImporter(workerConfig, {
|
|
84
83
|
initGenesisFromAncestry: options.initGenesisFromAncestry,
|
|
84
|
+
db: rootDb,
|
|
85
85
|
});
|
|
86
86
|
await importer.prepareForNextEpoch();
|
|
87
87
|
const api = {
|
|
@@ -6,6 +6,8 @@ import type { ImporterConfig, ImporterInternal } from "./protocol.js";
|
|
|
6
6
|
export type Config = WorkerConfig<ImporterConfig, BlocksDb, StatesDb<SerializedState<LeafDb>>>;
|
|
7
7
|
export type CreateImporterOptions = {
|
|
8
8
|
initGenesisFromAncestry?: boolean;
|
|
9
|
+
/** Provide an already-open database to reuse instead of opening a new one. */
|
|
10
|
+
db?: Awaited<ReturnType<Config["openDatabase"]>>;
|
|
9
11
|
};
|
|
10
12
|
export declare function createImporter(config: Config, options?: CreateImporterOptions): Promise<{
|
|
11
13
|
importer: Importer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtE,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtE,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,8EAA8E;IAC9E,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;CAClD,CAAC;AAEF,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC;IACT,QAAQ,EAAE,QAAQ,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;CACjD,CAAC,CA8BD;AAED;;;;;GAKG;AACH,wBAAsB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,iBAwCjE"}
|
|
@@ -10,7 +10,7 @@ const keccakHasher = keccak.KeccakHasher.create();
|
|
|
10
10
|
const blake2b = Blake2b.createHasher();
|
|
11
11
|
export async function createImporter(config, options = {}) {
|
|
12
12
|
const chainSpec = config.chainSpec;
|
|
13
|
-
const db = await config.openDatabase({ readonly: false });
|
|
13
|
+
const db = options.db ?? (await config.openDatabase({ readonly: false }));
|
|
14
14
|
const pvm = config.workerParams.pvm;
|
|
15
15
|
const blocks = db.getBlocksDb();
|
|
16
16
|
const states = db.getStatesDb();
|