@typeberry/lib 0.5.1-aff8b4f → 0.5.1-eb84786

package/packages/jam/node/main.js

@@ -0,0 +1,190 @@
+import { isMainThread } from "node:worker_threads";
+import { PvmBackend } from "#@typeberry/config";
+import { initWasm } from "#@typeberry/crypto";
+import { deriveBandersnatchSecretKey, deriveEd25519SecretKey, trivialSeed, } from "#@typeberry/crypto/key-derivation.js";
+import { Blake2b } from "#@typeberry/hash";
+import { ImporterConfig } from "#@typeberry/importer";
+import { NetworkingConfig } from "#@typeberry/jam-network";
+import { Listener } from "#@typeberry/listener";
+import { tryAsU16, tryAsU32 } from "#@typeberry/numbers";
+import { CURRENT_SUITE, CURRENT_VERSION, Result } from "#@typeberry/utils";
+import { DirectWorkerConfig } from "#@typeberry/workers-api";
+import { InMemWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
+import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
+import { initializeExtensions } from "./extensions.js";
+import * as metrics from "./metrics.js";
+import packageJson from "./package.json" with { type: "json" };
+import { spawnBlockGeneratorWorker, spawnImporterWorker, spawnNetworkWorker, startBlockGenerator, startImporterDirect, startNetwork, } from "./workers.js";
+export async function main(config, withRelPath, telemetry) {
+    if (!isMainThread) {
+        throw new Error("The main binary cannot be running as a Worker!");
+    }
+    await initWasm();
+    const nodeMetrics = metrics.createMetrics();
+    logger.info `🫐 Typeberry ${packageJson.version}. GP: ${CURRENT_VERSION} (${CURRENT_SUITE})`;
+    logger.info `🎸 Starting node: ${config.nodeName}.`;
+    logger.info `🖥️ PVM Backend: ${PvmBackend[config.pvmBackend]}.`;
+    const chainSpec = getChainSpec(config.node.flavor);
+    const blake2b = await Blake2b.createHasher();
+    const nodeName = config.nodeName;
+    const isInMemory = config.node.databaseBasePath === undefined;
+    const { dbPath, genesisHeaderHash } = getDatabasePath(blake2b, nodeName, config.node.chainSpec.genesisHeader, withRelPath(config.node.databaseBasePath ?? "<in-memory>"));
+    const baseConfig = { nodeName, chainSpec, blake2b, dbPath };
+    const importerParams = {
+        ...baseConfig,
+        workerParams: ImporterConfig.create({
+            pvm: config.pvmBackend,
+        }),
+    };
+    const importerConfig = isInMemory
+        ? { type: "inmem", config: InMemWorkerConfig.new(importerParams) }
+        : { type: "lmdb", config: LmdbWorkerConfig.new(importerParams) };
+    // Initialize the database with genesis state and block if there isn't one.
+    logger.info `🛢️ Opening database at ${dbPath}`;
+    const rootDb = importerConfig.config.openDatabase({ readonly: false });
+    await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry);
+    // NOTE [ToDr] even though, we should be closing the database here,
+    // it seems that opening it in the main thread for writing, and later
+    // in the importer thread, causes issues. Everything works fine though,
+    // if we DO NOT close the database (I guess it's process-shared?)
+    // await rootDb.close();
+    // Start block importer
+    let importer;
+    let closeImporter;
+    if (importerConfig.type === "inmem") {
+        ({ importer, finish: closeImporter } = await startImporterDirect(DirectWorkerConfig.new({
+            ...importerConfig.config,
+            blocksDb: rootDb.getBlocksDb(),
+            statesDb: rootDb.getStatesDb(),
+        })));
+    }
+    else {
+        ({ importer, finish: closeImporter } = await spawnImporterWorker(importerConfig.config));
+    }
+    const bestHeader = new Listener();
+    importer.setOnBestHeaderAnnouncement(async (header) => {
+        const slot = header.data.timeSlotIndex.materialize();
+        nodeMetrics.recordBestBlockChanged(slot, header.hash.toString());
+        await bestHeader.callbackHandler()(header);
+    });
+    // Start extensions
+    const closeExtensions = initializeExtensions({ chainSpec, bestHeader, nodeName });
+    // Authorship initialization.
+    // 1. load validator keys (bandersnatch, ed25519, bls)
+    // 2. allow the validator to specify metadata.
+    // 3. if we have validator keys, we should start the authorship module.
+    // NOTE: use trivialSeed to derive validator keys is safe
+    // because the authorship keys are only initialized when devValidatorIndex is specified (development mode),
+    // and trivial seeds are appropriate for test validators as defined in JIP-5.
+    const validatorIndex = config.devValidatorIndex ?? "all";
+    const authorshipKeys = {
+        keys: validatorIndex === "all"
+            ? Array.from({ length: chainSpec.validatorsCount })
+                .map((_, i) => trivialSeed(tryAsU32(i)))
+                .map((seed) => ({
+                bandersnatch: deriveBandersnatchSecretKey(seed, blake2b),
+                ed25519: deriveEd25519SecretKey(seed, blake2b),
+            }))
+            : [
+                {
+                    bandersnatch: deriveBandersnatchSecretKey(trivialSeed(tryAsU32(validatorIndex)), blake2b),
+                    ed25519: deriveEd25519SecretKey(trivialSeed(tryAsU32(validatorIndex)), blake2b),
+                },
+            ],
+    };
+    const closeAuthorship = await initAuthorship(importer, config.isAuthoring, rootDb, baseConfig, authorshipKeys, isInMemory);
+    // Networking initialization
+    const closeNetwork = await initNetwork(importer, rootDb, baseConfig, genesisHeaderHash, config.network, bestHeader, isInMemory);
+    const api = {
+        chainSpec,
+        async importBlock(block) {
+            const res = await importer.sendImportBlock(block);
+            if (res.isOk) {
+                return Result.ok(await importer.sendGetBestStateRootHash());
+            }
+            return res;
+        },
+        async getStateEntries(hash) {
+            return importer.sendGetStateEntries(hash);
+        },
+        async getBestStateRootHash() {
+            return importer.sendGetBestStateRootHash();
+        },
+        async close() {
+            logger.log `[main] ☠️ Closing the importer`;
+            await closeImporter();
+            logger.log `[main] ☠️  Closing the extensions`;
+            closeExtensions();
+            logger.log `[main] ☠️  Closing the authorship module`;
+            await closeAuthorship();
+            logger.log `[main] ☠️  Closing the networking module`;
+            await closeNetwork();
+            logger.log `[main] 🛢️ Closing the database`;
+            await rootDb.close();
+            logger.log `[main] 📳 Closing telemetry`;
+            await telemetry?.close();
+            logger.info `[main] ✅ Done.`;
+        },
+    };
+    return api;
+}
+const initAuthorship = async (importer, isAuthoring, rootDb, baseConfig, authorshipKeys, isInMemory) => {
+    if (!isAuthoring) {
+        logger.log `✍️  Authorship off: disabled`;
+        return () => Promise.resolve();
+    }
+    logger.info `✍️  Starting block generator.`;
+    const { generator, finish } = isInMemory
+        ? await startBlockGenerator(DirectWorkerConfig.new({
+            ...baseConfig,
+            blocksDb: rootDb.getBlocksDb(),
+            statesDb: rootDb.getStatesDb(),
+            workerParams: authorshipKeys,
+        }))
+        : await spawnBlockGeneratorWorker(LmdbWorkerConfig.new({
+            ...baseConfig,
+            workerParams: authorshipKeys,
+        }));
+    // relay blocks from generator to importer
+    generator.setOnBlock(async (block) => {
+        logger.log `✍️  Produced block at ${block.header.view().timeSlotIndex.materialize()}`;
+        await importer.sendImportBlock(block);
+    });
+    return finish;
+};
+const initNetwork = async (importer, rootDb, baseConfig, genesisHeaderHash, networkConfig, bestHeader, isInMemory) => {
+    if (networkConfig === null) {
+        logger.log `🛜 Networking off: no config`;
+        return () => Promise.resolve();
+    }
+    const { key, host, port, bootnodes } = networkConfig;
+    const networkingConfig = NetworkingConfig.create({
+        genesisHeaderHash,
+        key,
+        host,
+        port: tryAsU16(port),
+        bootnodes: bootnodes.map((node) => node.toString()),
+    });
+    const { network, finish } = isInMemory
+        ? await startNetwork(DirectWorkerConfig.new({
+            ...baseConfig,
+            blocksDb: rootDb.getBlocksDb(),
+            statesDb: rootDb.getStatesDb(),
+            workerParams: networkingConfig,
+        }))
+        : await spawnNetworkWorker(LmdbWorkerConfig.new({
+            ...baseConfig,
+            workerParams: networkingConfig,
+        }));
+    // relay blocks from networking to importer
+    network.setOnBlocks(async (newBlocks) => {
+        for (const block of newBlocks) {
+            await importer.sendImportBlock(block);
+        }
+    });
+    // relay newly imported headers to trigger network announcements
+    bestHeader.on((header) => {
+        network.sendNewHeader(header);
+    });
+    return finish;
+};

package/packages/jam/node/metrics.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Node-level metrics for JAM implementation.
+ *
+ * https://github.com/polkadot-fellows/JIPs/blob/main/JIP-3.md#status-events
+ */
+export declare function createMetrics(): {
+    recordBestBlockChanged(slot: number, headerHash: string): void;
+    recordFinalizedBlockChanged(slot: number, headerHash: string): void;
+    recordSyncStatusChanged(synced: boolean): void;
+};
+//# sourceMappingURL=metrics.d.ts.map

package/packages/jam/node/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/metrics.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AAEH,wBAAgB,aAAa;iCAwBI,MAAM,cAAc,MAAM,GAAG,IAAI;sCAI5B,MAAM,cAAc,MAAM,GAAG,IAAI;oCAInC,OAAO,GAAG,IAAI;EAIjD"}

package/packages/jam/node/metrics.js

@@ -0,0 +1,37 @@
+import { metrics } from "@opentelemetry/api";
+import packageJson from "./package.json" with { type: "json" };
+/**
+ * Node-level metrics for JAM implementation.
+ *
+ * https://github.com/polkadot-fellows/JIPs/blob/main/JIP-3.md#status-events
+ */
+export function createMetrics() {
+    const meter = metrics.getMeter(packageJson.name, packageJson.version);
+    // JIP-3
+    // 11
+    const bestBlockChangedCounter = meter.createCounter("jam.jip3.best_block_changed", {
+        description: "Best block changed events",
+        unit: "events",
+    });
+    // 12
+    const finalizedBlockChangedCounter = meter.createCounter("jam.jip3.finalized_block_changed", {
+        description: "Finalized block changed events",
+        unit: "events",
+    });
+    // 13
+    const syncStatusChangedCounter = meter.createCounter("jam.jip3.sync_status_changed", {
+        description: "Sync status changed events",
+        unit: "events",
+    });
+    return {
+        recordBestBlockChanged(slot, headerHash) {
+            bestBlockChangedCounter.add(1, { slot, header_hash: headerHash });
+        },
+        recordFinalizedBlockChanged(slot, headerHash) {
+            finalizedBlockChangedCounter.add(1, { slot, header_hash: headerHash });
+        },
+        recordSyncStatusChanged(synced) {
+            syncStatusChangedCounter.add(1, { synced });
+        },
+    };
+}

package/packages/jam/node/package.json

@@ -0,0 +1,39 @@
+{
+    "name": "@typeberry/node",
+    "version": "0.5.1",
+    "description": "The main typeberry node.",
+    "main": "index.ts",
+    "dependencies": {
+        "@opentelemetry/api": "1.9.0",
+        "@typeberry/block": "*",
+        "@typeberry/block-authorship": "*",
+        "@typeberry/block-json": "*",
+        "@typeberry/bytes": "*",
+        "@typeberry/codec": "*",
+        "@typeberry/collections": "*",
+        "@typeberry/config": "*",
+        "@typeberry/config-node": "*",
+        "@typeberry/configs": "*",
+        "@typeberry/crypto": "*",
+        "@typeberry/ext-ipc": "*",
+        "@typeberry/fuzz-proto": "*",
+        "@typeberry/hash": "*",
+        "@typeberry/importer": "*",
+        "@typeberry/jam-network": "*",
+        "@typeberry/json-parser": "*",
+        "@typeberry/listener": "*",
+        "@typeberry/logger": "*",
+        "@typeberry/numbers": "*",
+        "@typeberry/state-json": "*",
+        "@typeberry/state-merkleization": "*",
+        "@typeberry/utils": "*",
+        "@typeberry/workers-api": "*",
+        "@typeberry/workers-api-node": "*"
+    },
+    "scripts": {
+        "test": "tsx --test $(find . -type f -name '*.test.ts' | tr '\\n' ' ')"
+    },
+    "author": "Fluffy Labs",
+    "license": "MPL-2.0",
+    "type": "module"
+}

package/packages/jam/node/reader.d.ts

@@ -0,0 +1,141 @@
+import { Block } from "#@typeberry/block";
+import type { ChainSpec } from "#@typeberry/config";
+import { Logger } from "#@typeberry/logger";
+import type { NodeApi } from "./main.js";
+export type BlocksImporterConfig = {
+    /**
+     * Import blocks from a set of files (JSON vs bin - autodetected).
+     *
+     * The files containing blocks need to be ordered by the importing order.
+     */
+    files: string[];
+    /** chainspec config */
+    chainSpec: ChainSpec;
+};
+export declare const importBlocks: (node: NodeApi, blocksToImport: string[]) => Promise<void>;
+export declare function startBlocksReader(options: BlocksImporterConfig, logger: Logger): Generator<import("@typeberry/codec").ViewOf<Block, {
+    header: import("@typeberry/codec").Descriptor<import("@typeberry/block").Header & import("@typeberry/codec").CodecRecord<import("@typeberry/block").Header>, import("@typeberry/codec").ViewOf<import("@typeberry/block").Header & import("@typeberry/codec").CodecRecord<import("@typeberry/block").Header>, {
+        parentHeaderHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"HeaderHash">, import("@typeberry/bytes").Bytes<32>>;
+        priorStateRoot: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"StateRootHash">, import("@typeberry/bytes").Bytes<32>>;
+        extrinsicHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"ExtrinsicHash">, import("@typeberry/bytes").Bytes<32>>;
+        timeSlotIndex: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"TimeSlot[u32]">, import("@typeberry/bytes").Bytes<4>>;
+        epochMarker: import("@typeberry/codec").Descriptor<import("@typeberry/block").EpochMarker | null, import("@typeberry/codec").ViewOf<import("@typeberry/block").EpochMarker, {
+            entropy: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"EntropyHash">, import("@typeberry/bytes").Bytes<32>>;
+            ticketsEntropy: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"EntropyHash">, import("@typeberry/bytes").Bytes<32>>;
+            validators: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block").ValidatorKeys[] & import("@typeberry/utils").WithOpaque<"ValidatorsCount">, import("@typeberry/codec").SequenceView<import("@typeberry/block").ValidatorKeys, import("@typeberry/codec").ViewOf<import("@typeberry/block").ValidatorKeys, {
+                bandersnatch: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"BandersnatchKey">, import("@typeberry/bytes").Bytes<32>>;
+                ed25519: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"Ed25519Key">, import("@typeberry/bytes").Bytes<32>>;
+            }>>>;
+        }> | null>;
+        ticketsMarker: import("@typeberry/codec").Descriptor<import("@typeberry/block").TicketsMarker | null, import("@typeberry/codec").ViewOf<import("@typeberry/block").TicketsMarker, {
+            tickets: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block/tickets.js").Ticket[] & import("@typeberry/utils").WithOpaque<"EpochLength">, import("@typeberry/codec").SequenceView<import("@typeberry/block/tickets.js").Ticket, import("@typeberry/codec").ViewOf<import("@typeberry/block/tickets.js").Ticket, {
+                id: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32>, import("@typeberry/bytes").Bytes<32>>;
+                attempt: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<1> & import("@typeberry/utils").WithOpaque<"TicketAttempt[0|1|2]">, import("@typeberry/numbers").U8>;
+            }>>>;
+        }> | null>;
+        bandersnatchBlockAuthorIndex: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<2> & import("@typeberry/utils").WithOpaque<"ValidatorIndex[u16]">, import("@typeberry/bytes").Bytes<2>>;
+        entropySource: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<96> & import("@typeberry/utils").WithOpaque<"BandersnatchVrfSignature">, import("@typeberry/bytes").Bytes<96>>;
+        offendersMarker: import("@typeberry/codec").Descriptor<(import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"Ed25519Key">)[], import("@typeberry/codec").SequenceView<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"Ed25519Key">, import("@typeberry/bytes").Bytes<32>>>;
+        seal: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<96> & import("@typeberry/utils").WithOpaque<"BandersnatchVrfSignature">, import("@typeberry/bytes").Bytes<96>>;
+    }>>;
+    extrinsic: import("@typeberry/codec").Descriptor<import("@typeberry/block").Extrinsic, import("@typeberry/codec").ViewOf<import("@typeberry/block").Extrinsic, {
+        tickets: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block/tickets.js").SignedTicket[] & import("@typeberry/utils").WithOpaque<"Size: [0..chainSpec.maxTicketsPerExtrinsic)">, import("@typeberry/codec").SequenceView<import("@typeberry/block/tickets.js").SignedTicket, import("@typeberry/codec").ViewOf<import("@typeberry/block/tickets.js").SignedTicket, {
+            attempt: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<1> & import("@typeberry/utils").WithOpaque<"TicketAttempt[0|1|2]">, import("@typeberry/numbers").U8>;
+            signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<784> & import("@typeberry/utils").WithOpaque<"BandersnatchRingSignature">, import("@typeberry/bytes").Bytes<784>>;
+        }>>>;
+        preimages: import("@typeberry/codec").Descriptor<import("@typeberry/block/preimage.js").Preimage[], import("@typeberry/codec").SequenceView<import("@typeberry/block/preimage.js").Preimage, import("@typeberry/codec").ViewOf<import("@typeberry/block/preimage.js").Preimage, {
+            requester: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"ServiceId[u32]">, import("@typeberry/bytes").Bytes<4>>;
+            blob: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").BytesBlob, import("@typeberry/bytes").BytesBlob>;
+        }>>>;
+        guarantees: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block/guarantees.js").ReportGuarantee[] & import("@typeberry/utils").WithOpaque<"[0..CoresCount)">, import("@typeberry/codec").SequenceView<import("@typeberry/block/guarantees.js").ReportGuarantee, import("@typeberry/codec").ViewOf<import("@typeberry/block/guarantees.js").ReportGuarantee, {
+            report: import("@typeberry/codec").Descriptor<import("@typeberry/block/work-report.js").WorkReport, import("@typeberry/codec").ViewOf<import("@typeberry/block/work-report.js").WorkReport, {
+                workPackageSpec: import("@typeberry/codec").Descriptor<import("@typeberry/block/work-report.js").WorkPackageSpec, import("@typeberry/codec").ViewOf<import("@typeberry/block/work-report.js").WorkPackageSpec, {
+                    hash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkPackageHash">, import("@typeberry/bytes").Bytes<32>>;
+                    length: import("@typeberry/codec").Descriptor<import("@typeberry/numbers").U32, import("@typeberry/bytes").Bytes<4>>;
+                    erasureRoot: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32>, import("@typeberry/bytes").Bytes<32>>;
+                    exportsRoot: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"ExportsRootHash">, import("@typeberry/bytes").Bytes<32>>;
+                    exportsCount: import("@typeberry/codec").Descriptor<import("@typeberry/numbers").U16, import("@typeberry/bytes").Bytes<2>>;
+                }>>;
+                context: import("@typeberry/codec").Descriptor<import("@typeberry/block/refine-context.js").RefineContext, import("@typeberry/codec").ViewOf<import("@typeberry/block/refine-context.js").RefineContext, {
+                    anchor: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"HeaderHash">, import("@typeberry/bytes").Bytes<32>>;
+                    stateRoot: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"StateRootHash">, import("@typeberry/bytes").Bytes<32>>;
+                    beefyRoot: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"BeefyHash">, import("@typeberry/bytes").Bytes<32>>;
+                    lookupAnchor: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"HeaderHash">, import("@typeberry/bytes").Bytes<32>>;
+                    lookupAnchorSlot: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"TimeSlot[u32]">, import("@typeberry/bytes").Bytes<4>>;
+                    prerequisites: import("@typeberry/codec").Descriptor<(import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkPackageHash">)[], import("@typeberry/codec").SequenceView<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkPackageHash">, import("@typeberry/bytes").Bytes<32>>>;
+                }>>;
+                coreIndex: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<2> & import("@typeberry/utils").WithOpaque<"CoreIndex[u16]">, import("@typeberry/numbers").U32>;
+                authorizerHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"AuthorizerHash">, import("@typeberry/bytes").Bytes<32>>;
+                authorizationGasUsed: import("@typeberry/codec").Descriptor<bigint & import("@typeberry/numbers").WithBytesRepresentation<8> & import("@typeberry/utils").WithOpaque<"ServiceGas[u64]">, import("@typeberry/numbers").U64>;
+                authorizationOutput: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").BytesBlob, import("@typeberry/bytes").BytesBlob>;
+                segmentRootLookup: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block/refine-context.js").WorkPackageInfo[], import("@typeberry/codec").SequenceView<import("@typeberry/block/refine-context.js").WorkPackageInfo, import("@typeberry/codec").ViewOf<import("@typeberry/block/refine-context.js").WorkPackageInfo, {
+                    workPackageHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkPackageHash">, import("@typeberry/bytes").Bytes<32>>;
+                    segmentTreeRoot: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"ExportsRootHash">, import("@typeberry/bytes").Bytes<32>>;
+                }>>>;
+                results: import("@typeberry/codec").Descriptor<import("@typeberry/collections").FixedSizeArray<import("@typeberry/block/work-result.js").WorkResult, import("@typeberry/numbers").U8>, import("@typeberry/codec").SequenceView<import("@typeberry/block/work-result.js").WorkResult, import("@typeberry/codec").ViewOf<import("@typeberry/block/work-result.js").WorkResult, {
+                    serviceId: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"ServiceId[u32]">, import("@typeberry/bytes").Bytes<4>>;
+                    codeHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"CodeHash">, import("@typeberry/bytes").Bytes<32>>;
+                    payloadHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32>, import("@typeberry/bytes").Bytes<32>>;
+                    gas: import("@typeberry/codec").Descriptor<bigint & import("@typeberry/numbers").WithBytesRepresentation<8> & import("@typeberry/utils").WithOpaque<"ServiceGas[u64]">, import("@typeberry/bytes").Bytes<8>>;
+                    result: import("@typeberry/codec").Descriptor<import("@typeberry/block/work-result.js").WorkExecResult, {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.ok;
+                        okBlob: import("@typeberry/bytes").BytesBlob;
+                    } | {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.outOfGas;
+                    } | {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.panic;
+                    } | {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.incorrectNumberOfExports;
+                    } | {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.digestTooBig;
+                    } | {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.badCode;
+                    } | {
+                        kind: import("@typeberry/block/work-result.js").WorkExecResultKind.codeOversize;
+                    }>;
+                    load: import("@typeberry/codec").Descriptor<import("@typeberry/block/work-result.js").WorkRefineLoad, import("@typeberry/codec").ViewOf<import("@typeberry/block/work-result.js").WorkRefineLoad, {
+                        gasUsed: import("@typeberry/codec").Descriptor<bigint & import("@typeberry/numbers").WithBytesRepresentation<8> & import("@typeberry/utils").WithOpaque<"ServiceGas[u64]">, import("@typeberry/numbers").U64>;
+                        importedSegments: import("@typeberry/codec").Descriptor<import("@typeberry/numbers").U32, import("@typeberry/numbers").U32>;
+                        extrinsicCount: import("@typeberry/codec").Descriptor<import("@typeberry/numbers").U32, import("@typeberry/numbers").U32>;
+                        extrinsicSize: import("@typeberry/codec").Descriptor<import("@typeberry/numbers").U32, import("@typeberry/numbers").U32>;
+                        exportedSegments: import("@typeberry/codec").Descriptor<import("@typeberry/numbers").U32, import("@typeberry/numbers").U32>;
+                    }>>;
+                }>>>;
+            }>>;
+            slot: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"TimeSlot[u32]">, import("@typeberry/bytes").Bytes<4>>;
+            credentials: import("@typeberry/codec").Descriptor<(readonly import("@typeberry/block/guarantees.js").Credential[] & import("@typeberry/utils").WithOpaque<"2">) | (readonly import("@typeberry/block/guarantees.js").Credential[] & import("@typeberry/utils").WithOpaque<"3">), import("@typeberry/codec").SequenceView<import("@typeberry/block/guarantees.js").Credential, import("@typeberry/codec").ViewOf<import("@typeberry/block/guarantees.js").Credential, {
+                validatorIndex: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<2> & import("@typeberry/utils").WithOpaque<"ValidatorIndex[u16]">, import("@typeberry/bytes").Bytes<2>>;
+                signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<64> & import("@typeberry/utils").WithOpaque<"Ed25519Signature">, import("@typeberry/bytes").Bytes<64>>;
+            }>>>;
+        }>>>;
+        assurances: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block/assurances.js").AvailabilityAssurance[] & import("@typeberry/utils").WithOpaque<"[0 .. ValidatorsCount)">, import("@typeberry/codec").SequenceView<import("@typeberry/block/assurances.js").AvailabilityAssurance, import("@typeberry/codec").ViewOf<import("@typeberry/block/assurances.js").AvailabilityAssurance, {
+            anchor: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"HeaderHash">, import("@typeberry/bytes").Bytes<32>>;
+            bitfield: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").BitVec, import("@typeberry/bytes").BitVec>;
+            validatorIndex: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<2> & import("@typeberry/utils").WithOpaque<"ValidatorIndex[u16]">, import("@typeberry/bytes").Bytes<2>>;
+            signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<64> & import("@typeberry/utils").WithOpaque<"Ed25519Signature">, import("@typeberry/bytes").Bytes<64>>;
+        }>>>;
+        disputes: import("@typeberry/codec").Descriptor<import("@typeberry/block/disputes.js").DisputesExtrinsic, import("@typeberry/codec").ViewOf<import("@typeberry/block/disputes.js").DisputesExtrinsic, {
+            verdicts: import("@typeberry/codec").Descriptor<import("@typeberry/block/disputes.js").Verdict[], import("@typeberry/codec").SequenceView<import("@typeberry/block/disputes.js").Verdict, import("@typeberry/codec").ViewOf<import("@typeberry/block/disputes.js").Verdict, {
+                workReportHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkReportHash">, import("@typeberry/bytes").Bytes<32>>;
+                votesEpoch: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<4> & import("@typeberry/utils").WithOpaque<"Epoch">, import("@typeberry/bytes").Bytes<4>>;
+                votes: import("@typeberry/codec").Descriptor<readonly import("@typeberry/block/disputes.js").Judgement[] & import("@typeberry/utils").WithOpaque<"Validators super majority">, import("@typeberry/codec").SequenceView<import("@typeberry/block/disputes.js").Judgement, import("@typeberry/codec").ViewOf<import("@typeberry/block/disputes.js").Judgement, {
+                    isWorkReportValid: import("@typeberry/codec").Descriptor<boolean, boolean>;
+                    index: import("@typeberry/codec").Descriptor<number & import("@typeberry/numbers").WithBytesRepresentation<2> & import("@typeberry/utils").WithOpaque<"ValidatorIndex[u16]">, import("@typeberry/bytes").Bytes<2>>;
+                    signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<64> & import("@typeberry/utils").WithOpaque<"Ed25519Signature">, import("@typeberry/bytes").Bytes<64>>;
+                }>>>;
+            }>>>;
+            culprits: import("@typeberry/codec").Descriptor<import("@typeberry/block/disputes.js").Culprit[], import("@typeberry/codec").SequenceView<import("@typeberry/block/disputes.js").Culprit, import("@typeberry/codec").ViewOf<import("@typeberry/block/disputes.js").Culprit, {
+                workReportHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkReportHash">, import("@typeberry/bytes").Bytes<32>>;
+                key: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"Ed25519Key">, import("@typeberry/bytes").Bytes<32>>;
+                signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<64> & import("@typeberry/utils").WithOpaque<"Ed25519Signature">, import("@typeberry/bytes").Bytes<64>>;
+            }>>>;
+            faults: import("@typeberry/codec").Descriptor<import("@typeberry/block/disputes.js").Fault[], import("@typeberry/codec").SequenceView<import("@typeberry/block/disputes.js").Fault, import("@typeberry/codec").ViewOf<import("@typeberry/block/disputes.js").Fault, {
+                workReportHash: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"WorkReportHash">, import("@typeberry/bytes").Bytes<32>>;
+                wasConsideredValid: import("@typeberry/codec").Descriptor<boolean, boolean>;
+                key: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<32> & import("@typeberry/utils").WithOpaque<"Ed25519Key">, import("@typeberry/bytes").Bytes<32>>;
+                signature: import("@typeberry/codec").Descriptor<import("@typeberry/bytes").Bytes<64> & import("@typeberry/utils").WithOpaque<"Ed25519Signature">, import("@typeberry/bytes").Bytes<64>>;
+            }>>>;
+        }>>;
+    }>>;
+}>, void, any>;
+//# sourceMappingURL=reader.d.ts.map

package/packages/jam/node/reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/reader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAkB,MAAM,kBAAkB,CAAC;AAGzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,uBAAuB;IACvB,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AAEF,eAAO,MAAM,YAAY,GAAU,MAAM,OAAO,EAAE,gBAAgB,MAAM,EAAE,kBAsBzE,CAAC;AAEF,wBAAiB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAa/E"}

package/packages/jam/node/reader.js

@@ -0,0 +1,87 @@
+import fs from "node:fs";
+import { Block } from "#@typeberry/block";
+import { blockFromJson } from "#@typeberry/block-json";
+import { Decoder, Encoder, EndOfDataError } from "#@typeberry/codec";
+import { parseFromJson } from "#@typeberry/json-parser";
+import { Logger } from "#@typeberry/logger";
+import { resultToString } from "#@typeberry/utils";
+export const importBlocks = async (node, blocksToImport) => {
+    const logger = Logger.new(import.meta.filename, "jam");
+    logger.info `📖 Reading blocks from ${blocksToImport.length} files`;
+    const reader = startBlocksReader({
+        files: blocksToImport,
+        chainSpec: node.chainSpec,
+    }, logger);
+    for (const block of reader) {
+        logger.log `📖 Importing block: #${block.header.view().timeSlotIndex.materialize()}`;
+        const res = await node.importBlock(block);
+        if (res.isError) {
+            logger.error `📖 ${resultToString(res)}`;
+        }
+    }
+    // close the importer.
+    logger.info `All blocks scheduled to be imported.`;
+    return await node.close();
+};
+export function* startBlocksReader(options, logger) {
+    const { chainSpec } = options;
+    for (const file of options.files) {
+        const isJsonFile = isJson(file);
+        logger.log `📖 Reading ${file}...`;
+        if (isJsonFile) {
+            yield readJsonBlock(file, chainSpec);
+        }
+        else {
+            yield* readCodecBlocks(file, chainSpec);
+        }
+    }
+}
+const isJson = (f) => f.endsWith(".json");
+function* readCodecBlocks(file, chainSpec) {
+    const fileDescriptor = fs.openSync(file, "r");
+    try {
+        const bufferSize = 14 * 1024 * 1024; // should be at least max block length - https://graypaper.fluffylabs.dev/#/ab2cdbd/1b13001b1300?v=0.7.2
+        const buffer = new Uint8Array(bufferSize);
+        let offset = 0;
+        let bytesRead = 0;
+        let bytesRequested = bufferSize;
+        const getNextChunk = () => {
+            try {
+                bytesRead = fs.readSync(fileDescriptor, buffer, offset, bytesRequested, null);
+            }
+            catch (_) {
+                return false;
+            }
+            return bytesRead > 0;
+        };
+        while (getNextChunk()) {
+            let bytesDecoded = 0;
+            try {
+                const decoder = Decoder.fromBlob(buffer.subarray(0, offset + bytesRead), chainSpec);
+                while (true) {
+                    yield decoder.object(Block.Codec.View);
+                    bytesDecoded = decoder.bytesRead();
+                }
+            }
+            catch (e) {
+                if (!(e instanceof EndOfDataError)) {
+                    throw e;
+                }
+            }
+            bytesRequested = bytesDecoded;
+            buffer.set(buffer.subarray(bytesDecoded, offset + bytesRead), 0);
+            offset = offset + bytesRead - bytesDecoded;
+        }
+    }
+    finally {
+        fs.closeSync(fileDescriptor);
+    }
+}
+function readJsonBlock(file, chainSpec) {
+    const jsonData = fs.readFileSync(file, "utf-8");
+    const parsedData = JSON.parse(jsonData);
+    const blockData = "block" in parsedData ? parsedData.block : parsedData;
+    const block = parseFromJson(blockData, blockFromJson(chainSpec));
+    const encoded = Encoder.encodeObject(Block.Codec, block, chainSpec);
+    return Decoder.decodeObject(Block.Codec.View, encoded, chainSpec);
+}