@typeberry/lib 0.10.0-3e244fa → 0.10.0-4da4cc0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +31 -16
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +1 -1
- package/packages/workers/importer/main.d.ts +2 -0
- package/packages/workers/importer/main.d.ts.map +1 -1
- package/packages/workers/importer/main.js +1 -1
package/package.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAsBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8LrH"}
|
|
@@ -16,7 +16,7 @@ const logger = Logger.new(import.meta.filename, "fuzztarget");
|
|
|
16
16
|
const FUZZ_DB_SUBDIR = "typeberry-fuzz-db";
|
|
17
17
|
const FUZZ_DB_FJALL = "fjall-hybrid";
|
|
18
18
|
const FUZZ_DB_LMDB = "lmdb-hybrid";
|
|
19
|
-
const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, FUZZ_DB_LMDB];
|
|
19
|
+
const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, FUZZ_DB_LMDB, "fjall", "lmdb"];
|
|
20
20
|
/** Subdirectory (under the fuzzer's db dir) holding the reused fjall values keyspace. */
|
|
21
21
|
const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
|
|
22
22
|
/**
|
|
@@ -25,6 +25,8 @@ const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
|
|
|
25
25
|
* the resident memory bounded.
|
|
26
26
|
*/
|
|
27
27
|
const FUZZ_FJALL_CACHE_BYTES = 128 * 1024 * 1024;
|
|
28
|
+
/** Rebuild the fjall-hybrid values session every N resets to limit LSM read amplification. */
|
|
29
|
+
const REBUILD_FJALL_SESSION_EVERY = 50;
|
|
28
30
|
/**
|
|
29
31
|
* Resolve the directory the fuzzer should use for its on-disk database, or
|
|
30
32
|
* `undefined` for an in-memory database. The dedicated `FUZZ_DB_SUBDIR` is
|
|
@@ -75,6 +77,8 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
75
77
|
// every reset, because opening it is the slow part. Only the in-memory blocks
|
|
76
78
|
// and leaf sets are rebuilt for each vector. fjall-hybrid only.
|
|
77
79
|
let fjallSession = null;
|
|
80
|
+
// Track how many times resetState has been called for periodic fjall session rebuilds.
|
|
81
|
+
let resetCount = 0;
|
|
78
82
|
// Set when close() starts. Guards resetState so a fuzz command arriving
|
|
79
83
|
// mid-shutdown can't build a fresh node that close() then orphans.
|
|
80
84
|
let isClosing = false;
|
|
@@ -112,6 +116,8 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
112
116
|
runningNode = null;
|
|
113
117
|
await finish;
|
|
114
118
|
}
|
|
119
|
+
// Increment reset counter for periodic fjall session rebuilds.
|
|
120
|
+
resetCount++;
|
|
115
121
|
const buildNode = (databaseBasePath) => {
|
|
116
122
|
const isPersistent = databaseBasePath !== undefined;
|
|
117
123
|
return mainImporter({
|
|
@@ -141,33 +147,42 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
|
|
|
141
147
|
stateBackend: isPersistent ? hybridStateBackend : "lmdb",
|
|
142
148
|
// Reuse the session keyspace (fjall-hybrid only, other backends
|
|
143
149
|
// ignore it). Nothing to pass for the in-memory fallback.
|
|
144
|
-
sharedFjallSession:
|
|
150
|
+
sharedFjallSession: hybridStateBackend === "fjall-hybrid" ? (fjallSession ?? undefined) : undefined,
|
|
145
151
|
});
|
|
146
152
|
};
|
|
147
153
|
if (fuzzDbBase !== undefined) {
|
|
148
154
|
try {
|
|
149
155
|
if (hybridStateBackend === FUZZ_DB_FJALL) {
|
|
150
|
-
// fjall-hybrid:
|
|
151
|
-
//
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
// every reset, not from the values store, so reusing it does not
|
|
156
|
-
// resume the previous run.
|
|
157
|
-
if (fjallSession === null) {
|
|
158
|
-
// Start from a clean slate once, then keep the keyspace open.
|
|
156
|
+
// fjall-hybrid: manage a reused values session.
|
|
157
|
+
// Rebuild it periodically to avoid LSM read amplification.
|
|
158
|
+
const fjallSessionPath = `${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
|
|
159
|
+
if (resetCount === 1) {
|
|
160
|
+
// First reset: start from a clean slate.
|
|
159
161
|
await wipeFuzzDb(fuzzDbBase);
|
|
160
|
-
fjallSession = await FjallValuesSession.open(
|
|
162
|
+
fjallSession = await FjallValuesSession.open(fjallSessionPath, {
|
|
163
|
+
ephemeral: true,
|
|
164
|
+
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
165
|
+
});
|
|
166
|
+
logger.info `🗄️ Opened reusable fjall values session at ${fjallSessionPath}`;
|
|
167
|
+
}
|
|
168
|
+
if (resetCount % REBUILD_FJALL_SESSION_EVERY === 0 && fjallSession !== null) {
|
|
169
|
+
// Periodic rebuild: close, wipe session dir, and reopen.
|
|
170
|
+
const session = fjallSession;
|
|
171
|
+
fjallSession = null;
|
|
172
|
+
await session.close().catch(() => { });
|
|
173
|
+
await wipeFuzzDb(fjallSessionPath).catch(() => { });
|
|
174
|
+
}
|
|
175
|
+
if (fjallSession === null) {
|
|
176
|
+
// No active session: create a fresh one.
|
|
177
|
+
fjallSession = await FjallValuesSession.open(fjallSessionPath, {
|
|
161
178
|
ephemeral: true,
|
|
162
179
|
cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
|
|
163
180
|
});
|
|
164
|
-
logger.info `🗄️ Opened reusable fjall values session at ${
|
|
181
|
+
logger.info `🗄️ Opened reusable fjall values session at ${fjallSessionPath}`;
|
|
165
182
|
}
|
|
166
183
|
}
|
|
167
184
|
else {
|
|
168
|
-
//
|
|
169
|
-
// reset. A fresh db each reset makes `initializeDatabase` set up
|
|
170
|
-
// genesis again instead of resuming the previous run.
|
|
185
|
+
// All other backends ("fjall", "lmdb", "lmdb-hybrid"): wipe and reopen on every reset.
|
|
171
186
|
await wipeFuzzDb(fuzzDbBase);
|
|
172
187
|
}
|
|
173
188
|
runningNode = await buildNode(fuzzDbBase);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAoHlB"}
|
|
@@ -79,9 +79,9 @@ export async function mainImporter(config, withRelPath, options = {}) {
|
|
|
79
79
|
await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry, {
|
|
80
80
|
initGenesisFromAncestry: options.initGenesisFromAncestry,
|
|
81
81
|
});
|
|
82
|
-
await rootDb.close();
|
|
83
82
|
const { db, importer } = await createImporter(workerConfig, {
|
|
84
83
|
initGenesisFromAncestry: options.initGenesisFromAncestry,
|
|
84
|
+
db: rootDb,
|
|
85
85
|
});
|
|
86
86
|
await importer.prepareForNextEpoch();
|
|
87
87
|
const api = {
|
|
@@ -6,6 +6,8 @@ import type { ImporterConfig, ImporterInternal } from "./protocol.js";
|
|
|
6
6
|
export type Config = WorkerConfig<ImporterConfig, BlocksDb, StatesDb<SerializedState<LeafDb>>>;
|
|
7
7
|
export type CreateImporterOptions = {
|
|
8
8
|
initGenesisFromAncestry?: boolean;
|
|
9
|
+
/** Provide an already-open database to reuse instead of opening a new one. */
|
|
10
|
+
db?: Awaited<ReturnType<Config["openDatabase"]>>;
|
|
9
11
|
};
|
|
10
12
|
export declare function createImporter(config: Config, options?: CreateImporterOptions): Promise<{
|
|
11
13
|
importer: Importer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtE,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtE,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,8EAA8E;IAC9E,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;CAClD,CAAC;AAEF,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC;IACT,QAAQ,EAAE,QAAQ,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;CACjD,CAAC,CA8BD;AAED;;;;;GAKG;AACH,wBAAsB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,iBAwCjE"}
|
|
@@ -10,7 +10,7 @@ const keccakHasher = keccak.KeccakHasher.create();
|
|
|
10
10
|
const blake2b = Blake2b.createHasher();
|
|
11
11
|
export async function createImporter(config, options = {}) {
|
|
12
12
|
const chainSpec = config.chainSpec;
|
|
13
|
-
const db = await config.openDatabase({ readonly: false });
|
|
13
|
+
const db = options.db ?? (await config.openDatabase({ readonly: false }));
|
|
14
14
|
const pvm = config.workerParams.pvm;
|
|
15
15
|
const blocks = db.getBlocksDb();
|
|
16
16
|
const states = db.getStatesDb();
|