@typeberry/lib 0.5.1-aff8b4f → 0.5.1-eb84786

package/packages/workers/importer/importer.js

@@ -0,0 +1,180 @@
+import { tryAsTimeSlot } from "#@typeberry/block";
+import { WithHash } from "#@typeberry/hash";
+import { BlockVerifier, BlockVerifierError } from "#@typeberry/transition/block-verifier.js";
+import { DbHeaderChain, OnChain } from "#@typeberry/transition/chain-stf.js";
+import { measure, now, Result, resultToString } from "#@typeberry/utils";
+import * as metrics from "./metrics.js";
+export var ImporterErrorKind;
+(function (ImporterErrorKind) {
+    ImporterErrorKind[ImporterErrorKind["Verifier"] = 0] = "Verifier";
+    ImporterErrorKind[ImporterErrorKind["Stf"] = 1] = "Stf";
+    ImporterErrorKind[ImporterErrorKind["Update"] = 2] = "Update";
+})(ImporterErrorKind || (ImporterErrorKind = {}));
+const importerError = (kind, nested) => Result.taggedError(ImporterErrorKind, kind, nested);
+export class Importer {
+    hasher;
+    logger;
+    blocks;
+    states;
+    options;
+    verifier;
+    stf;
+    // TODO [ToDr] we cannot assume state reference does not change.
+    state;
+    // Hash of the block that we have the posterior state for in `state`.
+    currentHash;
+    metrics;
+    constructor(spec, pvm, hasher, logger, blocks, states, options = {}) {
+        this.hasher = hasher;
+        this.logger = logger;
+        this.blocks = blocks;
+        this.states = states;
+        this.options = options;
+        this.metrics = metrics.createMetrics();
+        const currentBestHeaderHash = this.blocks.getBestHeaderHash();
+        const state = states.getState(currentBestHeaderHash);
+        if (state === null) {
+            throw new Error(`Unable to load best state from header hash: ${currentBestHeaderHash}.`);
+        }
+        this.verifier = new BlockVerifier(hasher, blocks);
+        this.stf = new OnChain(spec, state, hasher, { pvm, accumulateSequentially: false }, DbHeaderChain.new(blocks));
+        this.state = state;
+        this.currentHash = currentBestHeaderHash;
+        this.prepareForNextEpoch();
+        logger.info `😎 Best time slot: ${state.timeslot} (header hash: ${currentBestHeaderHash})`;
+    }
+    /** Do some extra work for preparation for the next epoch. */
+    async prepareForNextEpoch() {
+        try {
+            await this.stf.prepareForNextEpoch();
+        }
+        catch (e) {
+            this.logger.error `Unable to prepare for next epoch: ${e}`;
+        }
+    }
+    // TODO [ToDr] import block and get state root
+    async importBlockWithStateRoot(block) {
+        const res = await this.importBlock(block);
+        if (res.isOk) {
+            return Result.ok(this.state.backend.getStateRoot(this.hasher.blake2b));
+        }
+        return res;
+    }
+    async importBlock(block) {
+        const timer = measure("importBlock");
+        const timeSlot = extractTimeSlot(block);
+        this.metrics.recordBlockImportingStarted(timeSlot);
+        const startTime = now();
+        const maybeBestHeader = await this.importBlockInternal(block);
+        const duration = now() - startTime;
+        if (maybeBestHeader.isOk) {
+            const bestHeader = maybeBestHeader.ok;
+            this.logger.info `🧊 Best block: #${timeSlot} (${bestHeader.hash})`;
+            this.logger.log `${timer()}`;
+            this.metrics.recordBlockImportComplete(duration, true);
+            return maybeBestHeader;
+        }
+        this.logger.log `❌ Rejected block #${timeSlot}: ${resultToString(maybeBestHeader)}`;
+        this.logger.log `${timer()}`;
+        this.metrics.recordBlockImportComplete(duration, false);
+        return maybeBestHeader;
+    }
+    async importBlockInternal(block) {
+        const logger = this.logger;
+        logger.log `🧱 Attempting to import a new block`;
+        const timerVerify = measure("import:verify");
+        const verifyStart = now();
+        const hash = await this.verifier.verifyBlock(block, {
+            skipParentAndStateRoot: this.options.initGenesisFromAncestry ?? false,
+        });
+        const verifyDuration = now() - verifyStart;
+        logger.log `${timerVerify()}`;
+        if (hash.isError) {
+            this.metrics.recordBlockVerificationFailed(resultToString(hash));
+            return importerError(ImporterErrorKind.Verifier, hash);
+        }
+        this.metrics.recordBlockVerified(verifyDuration);
+        // TODO [ToDr] This is incomplete/temporary fork support!
+        const parentHash = block.header.view().parentHeaderHash.materialize();
+        if (!this.currentHash.isEqualTo(parentHash)) {
+            const state = this.states.getState(parentHash);
+            if (state === null) {
+                const e = Result.error(BlockVerifierError.StateRootNotFound, () => `State not found for parent block ${parentHash}`);
+                if (!e.isError) {
+                    throw new Error("unreachable, just adding to make compiler happy");
+                }
+                return importerError(ImporterErrorKind.Verifier, e);
+            }
+            this.state.updateBackend(state?.backend);
+            this.prepareForNextEpoch();
+            this.currentHash = parentHash;
+        }
+        const timeSlot = block.header.view().timeSlotIndex.materialize();
+        const headerHash = hash.ok;
+        logger.log `🧱 Verified block: Got hash ${headerHash} for block at slot ${timeSlot}.`;
+        const timerStf = measure("import:stf");
+        const stfStart = now();
+        const res = await this.stf.transition(block, headerHash);
+        const stfDuration = now() - stfStart;
+        logger.log `${timerStf()}`;
+        if (res.isError) {
+            this.metrics.recordBlockExecutionFailed(resultToString(res));
+            return importerError(ImporterErrorKind.Stf, res);
+        }
+        this.metrics.recordBlockExecuted(stfDuration, 0);
+        // modify the state
+        const update = res.ok;
+        const timerState = measure("import:state");
+        const updateResult = await this.states.updateAndSetState(headerHash, this.state, update);
+        if (updateResult.isError) {
+            logger.error `🧱 Unable to update state: ${resultToString(updateResult)}`;
+            return importerError(ImporterErrorKind.Update, updateResult);
+        }
+        this.prepareForNextEpoch();
+        this.currentHash = headerHash;
+        logger.log `${timerState()}`;
+        // insert new state and the block to DB.
+        const timerDb = measure("import:db");
+        const writeBlocks = this.blocks.insertBlock(new WithHash(headerHash, block));
+        // Computation of the state root may happen asynchronously,
+        // but we still need to wait for it before next block can be imported
+        const stateRoot = await this.states.getStateRoot(this.state);
+        logger.log `🧱 Storing post-state-root for ${headerHash}: ${stateRoot}.`;
+        const writeStateRoot = this.blocks.setPostStateRoot(headerHash, stateRoot);
+        await Promise.all([writeBlocks, writeStateRoot]);
+        logger.log `${timerDb()}`;
+        // finally update the best block
+        await this.blocks.setBestHeaderHash(headerHash);
+        return Result.ok(new WithHash(headerHash, block.header.view()));
+    }
+    getBestStateRootHash() {
+        const bestHeaderHash = this.blocks.getBestHeaderHash();
+        const stateRoot = this.blocks.getPostStateRoot(bestHeaderHash);
+        return stateRoot;
+    }
+    getBestBlockHash() {
+        return this.blocks.getBestHeaderHash();
+    }
+    getStateEntries(headerHash) {
+        const state = this.states.getState(headerHash);
+        const stateEntries = state?.backend.intoStateEntries();
+        return stateEntries ?? null;
+    }
+    async close() {
+        await this.blocks.close();
+        await this.states.close();
+    }
+}
+/**
+ * Attempt to safely extract timeslot of a block.
+ *
+ * NOTE: it may fail if encoding is invalid.
+ */
+function extractTimeSlot(block) {
+    try {
+        return block.header.view().timeSlotIndex.materialize();
+    }
+    catch {
+        return tryAsTimeSlot(2 ** 32 - 1);
+    }
+}

package/packages/workers/importer/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./main.js";
+export * from "./metrics.js";
+export * from "./protocol.js";
+export declare const WORKER: import("url").URL;
+//# sourceMappingURL=index.d.ts.map

package/packages/workers/importer/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,eAAO,MAAM,MAAM,mBAAuD,CAAC"}

package/packages/workers/importer/index.js

@@ -0,0 +1,4 @@
+export * from "./main.js";
+export * from "./metrics.js";
+export * from "./protocol.js";
+export const WORKER = new URL("./bootstrap-importer.mjs", import.meta.url);

package/packages/workers/importer/main.d.ts

@@ -0,0 +1,21 @@
+import type { BlocksDb, LeafDb, StatesDb } from "#@typeberry/database";
+import type { SerializedState } from "#@typeberry/state-merkleization";
+import type { WorkerConfig } from "#@typeberry/workers-api";
+import { Importer } from "./importer.js";
+import type { ImporterConfig, ImporterInternal } from "./protocol.js";
+export type Config = WorkerConfig<ImporterConfig, BlocksDb, StatesDb<SerializedState<LeafDb>>>;
+export type CreateImporterOptions = {
+    initGenesisFromAncestry?: boolean;
+};
+export declare function createImporter(config: Config, options?: CreateImporterOptions): Promise<{
+    importer: Importer;
+    db: ReturnType<Config["openDatabase"]>;
+}>;
+/**
+ * The `BlockImporter` listens to `block` signals, where it expects
+ * RAW undecoded block objects (typically coming from the network).
+ *
+ * These blocks should be decoded, verified and later imported.
+ */
+export declare function main(config: Config, comms: ImporterInternal): Promise<void>;
+//# sourceMappingURL=main.d.ts.map

package/packages/workers/importer/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtE,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB,CAAC,EAAE,OAAO,CAAC;CACnC,CAAC;AAEF,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC;IACT,QAAQ,EAAE,QAAQ,CAAC;IACnB,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;CACxC,CAAC,CAcD;AAED;;;;;GAKG;AACH,wBAAsB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,iBAwCjE"}

package/packages/workers/importer/main.js

@@ -0,0 +1,60 @@
+import { initWasm } from "#@typeberry/crypto";
+import { Blake2b, keccak, ZERO_HASH } from "#@typeberry/hash";
+import { Logger } from "#@typeberry/logger";
+import { TransitionHasher } from "#@typeberry/transition";
+import { Result, resultToString } from "#@typeberry/utils";
+import { Importer } from "./importer.js";
+const logger = Logger.new(import.meta.filename, "importer");
+const keccakHasher = keccak.KeccakHasher.create();
+const blake2b = Blake2b.createHasher();
+export async function createImporter(config, options = {}) {
+    const chainSpec = config.chainSpec;
+    const db = config.openDatabase({ readonly: false });
+    const pvm = config.workerParams.pvm;
+    const blocks = db.getBlocksDb();
+    const states = db.getStatesDb();
+    const hasher = new TransitionHasher(await keccakHasher, await blake2b);
+    const importer = new Importer(chainSpec, pvm, hasher, logger, blocks, states, options);
+    return {
+        importer,
+        db,
+    };
+}
+/**
+ * The `BlockImporter` listens to `block` signals, where it expects
+ * RAW undecoded block objects (typically coming from the network).
+ *
+ * These blocks should be decoded, verified and later imported.
+ */
+export async function main(config, comms) {
+    const wasmPromise = initWasm();
+    logger.info `📥 Importer starting`;
+    const { importer, db } = await createImporter(config);
+    const finishPromise = new Promise((resolve) => {
+        comms.setOnFinish(async () => resolve());
+    });
+    comms.setOnImportBlock(async (block) => {
+        const res = await importer.importBlock(block);
+        if (res.isError) {
+            const errMsg = resultToString(res);
+            return Result.error(errMsg, () => errMsg);
+        }
+        await comms.sendBestHeaderAnnouncement(res.ok);
+        return Result.ok(res.ok.hash);
+    });
+    comms.setOnGetStateEntries(async (headerHash) => {
+        return importer.getStateEntries(headerHash);
+    });
+    comms.setOnGetBestStateRootHash(async () => {
+        return importer.getBestStateRootHash() ?? ZERO_HASH.asOpaque();
+    });
+    await wasmPromise;
+    logger.info `📥 Importer waiting for blocks.`;
+    // await finish signal
+    await finishPromise;
+    await importer.close();
+    logger.info `📥 Importer finished. Closing channel.`;
+    await db.close();
+    comms.destroy();
+    logger.info `📥 Importer 🪦`;
+}

package/packages/workers/importer/metrics.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Block importer metrics for JAM implementation.
+ *
+ * https://github.com/polkadot-fellows/JIPs/blob/main/JIP-3.md#block-authoringimporting-events
+ */
+export declare function createMetrics(): {
+    recordBlockImportComplete(totalDurationMs: number, success: boolean): void;
+    recordBlockImportingStarted(slot: number): void;
+    recordBlockVerificationFailed(reason: string): void;
+    recordBlockVerified(durationMs: number): void;
+    recordBlockExecutionFailed(reason: string): void;
+    recordBlockExecuted(durationMs: number, cost: number): void;
+};
+//# sourceMappingURL=metrics.d.ts.map

package/packages/workers/importer/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/importer/metrics.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AAEH,wBAAgB,aAAa;+CAwDkB,MAAM,WAAW,OAAO,GAAG,IAAI;sCAIxC,MAAM,GAAG,IAAI;0CAIT,MAAM,GAAG,IAAI;oCAInB,MAAM,GAAG,IAAI;uCAKV,MAAM,GAAG,IAAI;oCAIhB,MAAM,QAAQ,MAAM,GAAG,IAAI;EAM9D"}

package/packages/workers/importer/metrics.js

@@ -0,0 +1,75 @@
+import { metrics } from "@opentelemetry/api";
+import packageJson from "./package.json" with { type: "json" };
+/**
+ * Block importer metrics for JAM implementation.
+ *
+ * https://github.com/polkadot-fellows/JIPs/blob/main/JIP-3.md#block-authoringimporting-events
+ */
+export function createMetrics() {
+    const meter = metrics.getMeter(packageJson.name, packageJson.version);
+    const blockVerificationDuration = meter.createHistogram("jam.blockVerificationTime", {
+        description: "Duration of block verification",
+        unit: "ms",
+    });
+    const blockExecutionDuration = meter.createHistogram("jam.blockExecutionTime", {
+        description: "Duration of block execution",
+        unit: "ms",
+    });
+    const blockExecutionCost = meter.createHistogram("jam.blockExecutionGas", {
+        description: "Block execution cost (gas)",
+        unit: "gas",
+    });
+    const blockImportDuration = meter.createHistogram("jam.blockImportTime", {
+        description: "Total duration of block import (verification + execution)",
+        unit: "ms",
+    });
+    // JIP-3
+    // 43
+    const blockImportingCounter = meter.createCounter("jam.jip3.importing", {
+        description: "Block importing started",
+        unit: "blocks",
+    });
+    // 44
+    const blockVerificationFailedCounter = meter.createCounter("jam.jip3.verification_failed", {
+        description: "Block verification failed",
+        unit: "errors",
+    });
+    // 45
+    const blockVerifiedCounter = meter.createCounter("jam.jip3.verified", {
+        description: "Block verified successfully",
+        unit: "blocks",
+    });
+    // 46
+    const blockExecutionFailedCounter = meter.createCounter("jam.jip3.execution_failed", {
+        description: "Block execution failed",
+        unit: "errors",
+    });
+    // 47
+    const blockExecutedCounter = meter.createCounter("jam.jip3.executed", {
+        description: "Block executed successfully",
+        unit: "blocks",
+    });
+    return {
+        recordBlockImportComplete(totalDurationMs, success) {
+            blockImportDuration.record(totalDurationMs, { success });
+        },
+        recordBlockImportingStarted(slot) {
+            blockImportingCounter.add(1, { slot });
+        },
+        recordBlockVerificationFailed(reason) {
+            blockVerificationFailedCounter.add(1, { reason });
+        },
+        recordBlockVerified(durationMs) {
+            blockVerifiedCounter.add(1);
+            blockVerificationDuration.record(durationMs);
+        },
+        recordBlockExecutionFailed(reason) {
+            blockExecutionFailedCounter.add(1, { reason });
+        },
+        recordBlockExecuted(durationMs, cost) {
+            blockExecutedCounter.add(1);
+            blockExecutionDuration.record(durationMs);
+            blockExecutionCost.record(cost);
+        },
+    };
+}

package/packages/workers/importer/package.json

@@ -0,0 +1,33 @@
+{
+    "name": "@typeberry/importer",
+    "version": "0.5.1",
+    "description": "A JAM block importer queue.",
+    "main": "index.ts",
+    "dependencies": {
+        "@opentelemetry/api": "1.9.0",
+        "@typeberry/block": "*",
+        "@typeberry/bytes": "*",
+        "@typeberry/codec": "*",
+        "@typeberry/collections": "*",
+        "@typeberry/config": "*",
+        "@typeberry/config-node": "*",
+        "@typeberry/crypto": "*",
+        "@typeberry/database": "*",
+        "@typeberry/hash": "*",
+        "@typeberry/logger": "*",
+        "@typeberry/numbers": "*",
+        "@typeberry/ordering": "*",
+        "@typeberry/state-merkleization": "*",
+        "@typeberry/telemetry": "*",
+        "@typeberry/transition": "*",
+        "@typeberry/utils": "*",
+        "@typeberry/workers-api": "*",
+        "@typeberry/workers-api-node": "*"
+    },
+    "scripts": {
+        "test": "tsx --test $(find . -type f -name '*.test.ts' | tr '\\n' ' ')"
+    },
+    "author": "Fluffy Labs",
+    "license": "MPL-2.0",
+    "type": "module"
+}