@typeberry/lib 0.5.1-aff8b4f → 0.5.1-eb84786

package/packages/jam/jamnp-s/tasks/sync.js

@@ -0,0 +1,287 @@
+import { Header, tryAsTimeSlot, } from "#@typeberry/block";
+import { Encoder } from "#@typeberry/codec";
+import { WithHash } from "#@typeberry/hash";
+import { Logger } from "#@typeberry/logger";
+import { tryAsU32 } from "#@typeberry/numbers";
+import { assertNever, OK } from "#@typeberry/utils";
+import { BlockSequenceError, handleGetBlockSequence } from "../protocol/ce-128-block-request.js";
+import { ce128, up0 } from "../protocol/index.js";
+import { handleAsyncErrors } from "../utils.js";
+export const SYNC_AUX = {
+    id: Symbol("sync"),
+};
+const logger = Logger.new(import.meta.filename, "net:sync");
+/**
+ * Maximal number of blocks we will send as a response to CE128.
+ */
+const MAX_BLOCK_SEQUENCE = 128;
+export class SyncTask {
+    spec;
+    blake2b;
+    streamManager;
+    connections;
+    blocks;
+    onNewBlocks;
+    static start(spec, blake2b, streamManager, connections, blocks, 
+    // TODO [ToDr] Use listener instead of a callback maybe?
+    onNewBlocks) {
+        const syncTask = new SyncTask(spec, blake2b, streamManager, connections, blocks, onNewBlocks);
+        const getPeerForStream = (streamId) => {
+            // NOTE [ToDr] Needing to query stream manager for a peer might be a bit
+            // wasteful, since we probably know the peer when we dispatch the
+            // stream message, however it's nice that the current abstraction of
+            // streams does not know anything about peers. Revisit if it gets ugly.
+            // retrieve a peer for that stream
+            return streamManager.getPeer(streamId);
+        };
+        const up0Handler = new up0.Handler(spec, () => syncTask.getUp0Handshake(), (streamId, ann) => {
+            const peer = getPeerForStream(streamId);
+            if (peer !== null) {
+                syncTask.onUp0Annoucement(peer, ann);
+            }
+        }, (streamId, handshake) => {
+            const peer = getPeerForStream(streamId);
+            if (peer !== null) {
+                syncTask.onUp0Handshake(peer, handshake);
+            }
+        });
+        // server mode
+        streamManager.registerIncomingHandlers(up0Handler);
+        streamManager.registerIncomingHandlers(new ce128.ServerHandler(spec, (streamId, hash, direction, maxBlocks) => {
+            const peer = getPeerForStream(streamId);
+            if (peer !== null) {
+                return syncTask.handleGetBlockSequence(peer, hash, direction, maxBlocks);
+            }
+            return [];
+        }));
+        // client mode
+        streamManager.registerOutgoingHandlers(up0Handler);
+        streamManager.registerOutgoingHandlers(new ce128.ClientHandler(spec));
+        return syncTask;
+    }
+    // Other's best header hash with timeslot
+    othersBest;
+    constructor(spec, blake2b, streamManager, connections, blocks, onNewBlocks) {
+        this.spec = spec;
+        this.blake2b = blake2b;
+        this.streamManager = streamManager;
+        this.connections = connections;
+        this.blocks = blocks;
+        this.onNewBlocks = onNewBlocks;
+        const ourBestHash = blocks.getBestHeaderHash();
+        // Get best block view
+        const ourBestBlock = blocks.getHeader(ourBestHash);
+        if (ourBestBlock === null) {
+            throw new Error(`Best header ${ourBestHash} missing in the database?`);
+        }
+        this.othersBest = up0.HashAndSlot.create({
+            hash: ourBestHash,
+            slot: ourBestBlock.timeSlotIndex.materialize(),
+        });
+    }
+    onUp0Handshake(peer, handshake) {
+        const { hash, slot } = handshake.final;
+        this.connections.withAuxData(peer.id, SYNC_AUX, (aux) => {
+            if (aux === undefined) {
+                return {
+                    finalBlockHash: hash,
+                    finalBlockSlot: slot,
+                    bestHeader: null,
+                };
+            }
+            aux.finalBlockHash = hash;
+            aux.finalBlockSlot = slot;
+            aux.bestHeader = null;
+            return aux;
+        });
+        if (this.othersBest.slot < slot) {
+            this.othersBest = handshake.final;
+        }
+    }
+    onUp0Annoucement(peer, announcement) {
+        const { hash, slot } = announcement.final;
+        const bestHeader = hashHeader(this.blake2b, announcement.header, this.spec);
+        logger.info `[${peer.id}] --> Received new header #${announcement.header.timeSlotIndex}: ${bestHeader.hash}`;
+        // NOTE [ToDr] Instead of having `Connections` store aux data perhaps
+        // we should maintain that directly? However that would require
+        // listening to peers connected/disconnected to perfrom some cleanups
+        // and extra persistence.
+        //
+        // update the peer info
+        this.connections.withAuxData(peer.id, SYNC_AUX, (aux) => {
+            if (aux === undefined) {
+                return {
+                    finalBlockHash: hash,
+                    finalBlockSlot: slot,
+                    bestHeader,
+                };
+            }
+            aux.finalBlockHash = hash;
+            aux.finalBlockSlot = slot;
+            aux.bestHeader = bestHeader;
+            return aux;
+        });
+        // TODO [ToDr] This should take finality into account, which would
+        // also indirectly do ancestry checks (i.e. we assume that the peer
+        // is verifying that the best block is built on top of it's own
+        // reported finalized block).
+        //
+        // now check if we should sync that block
+        if (this.othersBest.slot < bestHeader.data.timeSlotIndex) {
+            this.othersBest = up0.HashAndSlot.create({
+                hash: bestHeader.hash,
+                slot: bestHeader.data.timeSlotIndex,
+            });
+        }
+    }
+    getUp0Handshake() {
+        // TODO [ToDr] We don't have finality yet,
+        // we just treat each produced block as instantly-finalized.
+        const bestBlockHash = this.blocks.getBestHeaderHash();
+        const bestHeader = this.blocks.getHeader(bestBlockHash);
+        const timeSlot = bestHeader?.timeSlotIndex.materialize();
+        const bestBlock = up0.HashAndSlot.create({
+            hash: bestBlockHash,
+            slot: timeSlot ?? tryAsTimeSlot(0),
+        });
+        return up0.Handshake.create({
+            final: bestBlock,
+            leafs: [],
+        });
+    }
+    /**
+     * Open a UP0 stream with given peer.
+     *
+     * This will automatically send a handshake as well.
+     */
+    openUp0(peer) {
+        this.streamManager.withNewStream(peer, up0.STREAM_KIND, (handler, sender) => {
+            handler.sendHandshake(sender);
+            return OK;
+        });
+    }
+    /** Broadcast header we have seen or produced to our peers. */
+    // TODO [ToDr] consider returning promise for backpressure
+    broadcastHeader(header) {
+        const slot = header.data.timeSlotIndex.materialize();
+        const annoucement = up0.Announcement.create({
+            header: header.data.materialize(),
+            final: up0.HashAndSlot.create({
+                hash: header.hash,
+                slot,
+            }),
+        });
+        // TODO [ToDr] we currently gossip to everyone, but we probably should:
+        // 1. Gossip to peers in batches (sqrt(n) peers first?)
+        // 2. Gossip only to the peers that don't know about that header yet.
+        const peers = this.connections.getConnectedPeers();
+        for (const peerInfo of peers) {
+            this.streamManager.withStreamOfKind(peerInfo.peerId, up0.STREAM_KIND, (handler, sender) => {
+                logger.log `[${peerInfo.peerId}] <-- Broadcasting new header #${slot}: ${header.hash}`;
+                handler.sendAnnouncement(sender, annoucement);
+                return OK;
+            });
+        }
+    }
+    handleGetBlockSequence(peer, startHash, direction, maxBlocks) {
+        const limit = tryAsU32(Math.min(maxBlocks, MAX_BLOCK_SEQUENCE));
+        const res = handleGetBlockSequence(this.spec, this.blocks, startHash, direction, limit);
+        if (res.isOk) {
+            return res.ok;
+        }
+        if (res.error === BlockSequenceError.BlockOnFork) {
+            // seems that peer is requesting syncing a fork from us, let's bail.
+            logger.warn `[${peer.id}] --> Invalid block sequence request: ${startHash} is on a fork.`;
+            return [];
+        }
+        if (res.error === BlockSequenceError.NoStartBlock) {
+            // we don't know about that block at all, so let's just bail.
+            // we should probably penalize the peer for sending BS?
+            logger.warn `[${peer.id}] --> Invalid block sequence request: ${startHash} missing header or extrinsic.`;
+            return [];
+        }
+        assertNever(res.error);
+    }
+    /** Should be called periodically to request best seen blocks from other peers. */
+    maintainSync() {
+        // figure out where we are at
+        const ourBestHash = this.blocks.getBestHeaderHash();
+        const ourBestHeader = this.blocks.getHeader(ourBestHash);
+        const peerCount = this.connections.getPeerCount();
+        if (ourBestHeader === null) {
+            return {
+                kind: SyncResult.OurBestHeaderMissing,
+            };
+        }
+        const ourBestSlot = ourBestHeader.timeSlotIndex.materialize();
+        // figure out where others are at
+        const othersBest = this.othersBest;
+        const blocksToSync = othersBest.slot - ourBestSlot;
+        logger.trace `Our best. ${ourBestSlot}. Best seen: ${othersBest.slot}`;
+        if (blocksToSync < 1) {
+            this.connections.getPeerCount();
+            logger.trace `No new blocks. ${peerCount} peers.`;
+            return {
+                kind: SyncResult.NoNewBlocks,
+                ours: ourBestSlot,
+                theirs: othersBest.slot,
+            };
+        }
+        const requested = [];
+        logger.log `Sync ${blocksToSync} blocks from ${peerCount} peers.`;
+        // NOTE [ToDr] We might be requesting the same blocks from many peers
+        // which isn't very optimal, but for now: 🤷
+        //
+        // find peers that might have that block
+        for (const peerInfo of this.connections.getConnectedPeers()) {
+            const auxData = this.connections.getAuxData(peerInfo.peerId, SYNC_AUX);
+            // no aux data for that peer or peer not connected?
+            if (auxData === undefined || peerInfo.peerRef === null) {
+                continue;
+            }
+            const bestSlot = auxData.bestHeader !== null ? auxData.bestHeader.data.timeSlotIndex : auxData.finalBlockSlot;
+            const bestHash = auxData.bestHeader !== null ? auxData.bestHeader.hash : auxData.finalBlockHash;
+            // the peer doesn't have anything new for us
+            if (bestSlot <= ourBestSlot) {
+                continue;
+            }
+            // add some details for statistics.
+            requested.push({
+                peerId: peerInfo.peerId,
+                theirs: bestSlot,
+                count: bestSlot - ourBestSlot,
+            });
+            // request as much blocks from that peer as possible.
+            this.streamManager.withNewStream(peerInfo.peerRef, ce128.STREAM_KIND, (handler, sender) => {
+                handleAsyncErrors(async () => {
+                    logger.log `[${peerInfo.peerId}] <-- Fetching ${bestSlot - ourBestSlot} blocks (${bestHash})`;
+                    const blocks = await handler.requestBlockSequence(sender, bestHash, ce128.Direction.DescIncl, tryAsU32(bestSlot - ourBestSlot));
+                    blocks.reverse();
+                    this.onNewBlocks(blocks, peerInfo.peerId);
+                }, (e) => {
+                    logger.warn `[${peerInfo.peerId}] <-- requesting blocks to import: ${e}`;
+                });
+                return OK;
+            });
+        }
+        return {
+            kind: SyncResult.BlocksRequested,
+            ours: ourBestSlot,
+            requested,
+        };
+    }
+}
+/** Some extra details about how maintaining sync went. */
+export var SyncResult;
+(function (SyncResult) {
+    /** We didn't find our best header? */
+    SyncResult[SyncResult["OurBestHeaderMissing"] = 1] = "OurBestHeaderMissing";
+    /** There is no new blocks that we can sync. */
+    SyncResult[SyncResult["NoNewBlocks"] = 2] = "NoNewBlocks";
+    /** Sent request to some peers. */
+    SyncResult[SyncResult["BlocksRequested"] = 3] = "BlocksRequested";
+})(SyncResult || (SyncResult = {}));
+function hashHeader(blake2b, header, spec) {
+    const encoded = Encoder.encodeObject(Header.Codec, header, spec);
+    return new WithHash(blake2b.hashBytes(encoded).asOpaque(), header);
+}

package/packages/jam/jamnp-s/tasks/sync.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=sync.test.d.ts.map

package/packages/jam/jamnp-s/tasks/sync.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync.test.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/sync.test.ts"],"names":[],"mappings":""}

package/packages/jam/jamnp-s/tasks/sync.test.js

@@ -0,0 +1,227 @@
+import assert, { deepEqual } from "node:assert";
+import { before, describe, it } from "node:test";
+import { setTimeout } from "node:timers/promises";
+import { Block, Header, tryAsTimeSlot, } from "#@typeberry/block";
+import { testBlockView } from "#@typeberry/block/test-helpers.js";
+import { Bytes } from "#@typeberry/bytes";
+import { Decoder, Encoder } from "#@typeberry/codec";
+import { tinyChainSpec } from "#@typeberry/config";
+import { InMemoryBlocks } from "#@typeberry/database";
+import { Blake2b, HASH_SIZE, WithHash } from "#@typeberry/hash";
+import { Logger } from "#@typeberry/logger";
+import { createTestPeerPair, MockNetwork } from "#@typeberry/networking/testing.js";
+import { setupPeerListeners } from "../network.js";
+import { Connections } from "../peers.js";
+import { StreamManager } from "../stream-manager.js";
+import { SyncResult, SyncTask } from "./sync.js";
+const logger = Logger.new(import.meta.filename, "test:net");
+const spec = tinyChainSpec;
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+const toBlockView = (block) => {
+    const encodedBlock = Encoder.encodeObject(Block.Codec, block, spec);
+    const blockView = Decoder.decodeObject(Block.Codec.View, encodedBlock, spec);
+    return blockView;
+};
+const toHeaderView = (header) => {
+    const encodedHeader = Encoder.encodeObject(Header.Codec, header, spec);
+    const headerView = Decoder.decodeObject(Header.Codec.View, encodedHeader, spec);
+    return headerView;
+};
+describe("SyncTask", () => {
+    async function init(name, ourBlocks = []) {
+        const network = new MockNetwork(name);
+        const streamManager = new StreamManager();
+        const connections = new Connections(network);
+        const blocksDb = await setupTestDatabase(ourBlocks);
+        const receivedBlocks = [];
+        const onNewBlocks = (blocks) => {
+            receivedBlocks.push(blocks.map((view) => view.materialize()));
+        };
+        const syncTask = SyncTask.start(spec, blake2b, streamManager, connections, blocksDb, onNewBlocks);
+        setupPeerListeners(syncTask, network, streamManager);
+        let connectionIdx = 0;
+        const openConnection = (other) => {
+            // we need to create a pair of peers that connected together
+            const [self, peer1] = createTestPeerPair(connectionIdx++, name, other.name);
+            network._peers.peerConnected(peer1);
+            other.network._peers.peerConnected(self);
+            return [self, peer1];
+        };
+        return {
+            name,
+            syncTask,
+            network,
+            receivedBlocks,
+            connections,
+            openConnection,
+        };
+    }
+    async function tick() {
+        logger.log `tick`;
+        // TODO [ToDr] This is pretty imperfect. We basically need some way,
+        // to let the background reading tasks to process incoming data.
+        // Might be good enough? 🤷
+        await setTimeout(1);
+    }
+    it("should maintain sync with no peers", async () => {
+        // Setup
+        const { syncTask } = await init("self", blocksSeq({ start: 5 }));
+        const result = syncTask.maintainSync();
+        deepEqual(result, {
+            kind: SyncResult.NoNewBlocks,
+            ours: tryAsTimeSlot(5),
+            // we assume that our block is the best seen so far.
+            theirs: tryAsTimeSlot(5),
+        });
+    });
+    it("should sync with one peer", async () => {
+        const self = await init("self", blocksSeq({ start: 5, end: 7 }));
+        const peer1 = await init("peer1", blocksSeq({ start: 5, end: 10 }));
+        self.openConnection(peer1);
+        await tick();
+        const resultPeer = peer1.syncTask.maintainSync();
+        deepEqual(resultPeer, {
+            kind: SyncResult.NoNewBlocks,
+            ours: tryAsTimeSlot(10),
+            // we assume that our block is the best seen so far.
+            theirs: tryAsTimeSlot(10),
+        });
+        await tick();
+        const result = self.syncTask.maintainSync();
+        deepEqual(result, {
+            kind: SyncResult.BlocksRequested,
+            ours: tryAsTimeSlot(7),
+            requested: [
+                {
+                    count: 3,
+                    peerId: "peer1",
+                    theirs: 10,
+                },
+            ],
+        });
+        await tick();
+        deepEqual(self.receivedBlocks[0].map((x) => x.header.timeSlotIndex), [7, 8, 9, 10]);
+    });
+    it("should sync with multiple peers", async () => {
+        const self = await init("self", blocksSeq({ start: 5, end: 7 }));
+        const peer1 = await init("peer1", blocksSeq({ start: 5, end: 10 }));
+        const peer2 = await init("peer2", blocksSeq({ start: 5, end: 12 }));
+        self.openConnection(peer1);
+        self.openConnection(peer2);
+        await tick();
+        const resultSelf = self.syncTask.maintainSync();
+        deepEqual(resultSelf, {
+            kind: SyncResult.BlocksRequested,
+            ours: tryAsTimeSlot(7),
+            requested: [
+                {
+                    count: 3,
+                    peerId: "peer1",
+                    theirs: 10,
+                },
+                {
+                    count: 5,
+                    peerId: "peer2",
+                    theirs: 12,
+                },
+            ],
+        });
+        await tick();
+        deepEqual(self.receivedBlocks.map((chunk) => chunk.map((block) => block.header.timeSlotIndex)), [
+            [7, 8, 9, 10],
+            [7, 8, 9, 10, 11, 12],
+        ]);
+    });
+    it("should broadcast our header to one connected peers", async () => {
+        await broadcastTest(1);
+    });
+    it("should broadcast our header to two connected peers", async () => {
+        await broadcastTest(2);
+    });
+    it("should broadcast our header to multiple connected peers", async () => {
+        await broadcastTest(10);
+    });
+    async function broadcastTest(peersCount) {
+        const blocks = blocksSeq({ start: 0, end: 1 });
+        const newBlock = blocks.pop();
+        assert.ok(newBlock !== undefined);
+        const self = await init("self", blocks);
+        const peers = await Promise.all(Array.from({ length: peersCount }).map((_v, id) => {
+            return init(`peer${id}`, blocks);
+        }));
+        for (const p of peers) {
+            self.openConnection(p);
+        }
+        await tick();
+        for (const p of peers) {
+            deepEqual(p.syncTask.maintainSync().kind, SyncResult.NoNewBlocks);
+        }
+        // Send broadcast
+        self.syncTask.broadcastHeader(new WithHash(newBlock.hash, toHeaderView(newBlock.data.header)));
+        await tick();
+        for (const p of peers) {
+            deepEqual(p.syncTask.maintainSync(), {
+                kind: SyncResult.BlocksRequested,
+                ours: tryAsTimeSlot(0),
+                requested: [
+                    {
+                        count: 1,
+                        peerId: "self",
+                        theirs: tryAsTimeSlot(1),
+                    },
+                ],
+            });
+        }
+    }
+});
+function blocksSeq({ start, end = start }) {
+    if (start > end) {
+        throw new Error(`No blocks to create: ${start} > ${end}`);
+    }
+    const blocks = [];
+    for (let i = start; i <= end; i++) {
+        const prev = blocks.length > 0 ? blocks[blocks.length - 1] : null;
+        const parentHash = prev?.hash ?? Bytes.zero(HASH_SIZE).asOpaque();
+        blocks.push(createTestBlock({
+            parentHash,
+            timeSlot: tryAsTimeSlot(i),
+        }));
+    }
+    return blocks;
+}
+function createTestBlock(options = {}) {
+    const baseBlock = testBlockView().materialize();
+    const header = Header.create({
+        ...baseBlock.header,
+        parentHeaderHash: options.parentHash ?? baseBlock.header.parentHeaderHash,
+        timeSlotIndex: options.timeSlot ?? baseBlock.header.timeSlotIndex,
+        priorStateRoot: options.stateRoot ?? baseBlock.header.priorStateRoot,
+    });
+    const block = Block.create({ ...baseBlock, header });
+    const view = toBlockView(block);
+    const headerHash = blake2b.hashBytes(view.header.encoded());
+    return new WithHash(headerHash.asOpaque(), block);
+}
+async function setupTestDatabase(inBlocks = []) {
+    const db = InMemoryBlocks.new();
+    let blocks = inBlocks;
+    if (blocks.length === 0) {
+        // Create a genesis block by default
+        const genesisBlock = createTestBlock({ timeSlot: tryAsTimeSlot(0) });
+        blocks = [genesisBlock];
+    }
+    // Insert blocks and set the last one as best
+    for (let i = 0; i < blocks.length; i++) {
+        const block = blocks[i];
+        const blockView = toBlockView(block.data);
+        await db.insertBlock(new WithHash(block.hash, blockView));
+    }
+    // Set the last block as best
+    if (blocks.length > 0) {
+        await db.setBestHeaderHash(blocks[blocks.length - 1].hash);
+    }
+    return db;
+}

package/packages/jam/jamnp-s/utils.d.ts

@@ -0,0 +1,2 @@
+export declare function handleAsyncErrors(work: () => Promise<void>, onError: (e: unknown) => void): Promise<void>;
+//# sourceMappingURL=utils.d.ts.map

package/packages/jam/jamnp-s/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/jamnp-s/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,iBAEzF"}

package/packages/jam/jamnp-s/utils.js

@@ -0,0 +1,3 @@
+export function handleAsyncErrors(work, onError) {
+    return work().catch(onError);
+}

package/packages/jam/node/common.d.ts

@@ -0,0 +1,23 @@
+import { type HeaderHash, type TimeSlot } from "#@typeberry/block";
+import { Bytes, type BytesBlob } from "#@typeberry/bytes";
+import { type ChainSpec } from "#@typeberry/config";
+import { type JipChainSpec, KnownChainSpec } from "#@typeberry/config-node";
+import type { BlocksDb, RootDb, SerializedStatesDb } from "#@typeberry/database";
+import { type Blake2b } from "#@typeberry/hash";
+import { Logger } from "#@typeberry/logger";
+export declare const logger: Logger;
+export declare function getChainSpec(name: KnownChainSpec): ChainSpec;
+export declare function getDatabasePath(blake2b: Blake2b, nodeName: string, genesisHeader: BytesBlob, databaseBasePath: string): {
+    dbPath: string;
+    genesisHeaderHash: Bytes<32> & import("@typeberry/utils").WithOpaque<"HeaderHash">;
+};
+/**
+ * Initialize the database unless it's already initialized.
+ *
+ * The function checks the genesis header
+ */
+export type InitDbOptions = {
+    initGenesisFromAncestry?: boolean;
+};
+export declare function initializeDatabase(spec: ChainSpec, blake2b: Blake2b, genesisHeaderHash: HeaderHash, rootDb: RootDb<BlocksDb, SerializedStatesDb>, config: JipChainSpec, ancestry: [HeaderHash, TimeSlot][], options?: InitDbOptions): Promise<void>;
+//# sourceMappingURL=common.d.ts.map

package/packages/jam/node/common.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,QAAQ,EACd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,KAAK,SAAS,EAAgC,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,KAAK,YAAY,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,KAAK,OAAO,EAAuB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAG3C,eAAO,MAAM,MAAM,QAA0C,CAAC;AAE9D,wBAAgB,YAAY,CAAC,IAAI,EAAE,cAAc,aAUhD;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,SAAS,EACxB,gBAAgB,EAAE,MAAM;;;EAWzB;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,uBAAuB,CAAC,EAAE,OAAO,CAAC;CACnC,CAAC;AAEF,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,OAAO,EAChB,iBAAiB,EAAE,UAAU,EAC7B,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC,EAC5C,MAAM,EAAE,YAAY,EACpB,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,EAClC,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAuCf"}

package/packages/jam/node/common.js

@@ -0,0 +1,71 @@
+import { Block, emptyBlock, Header, reencodeAsView, } from "#@typeberry/block";
+import { Bytes } from "#@typeberry/bytes";
+import { Decoder } from "#@typeberry/codec";
+import { fullChainSpec, tinyChainSpec } from "#@typeberry/config";
+import { KnownChainSpec } from "#@typeberry/config-node";
+import { HASH_SIZE, WithHash } from "#@typeberry/hash";
+import { Logger } from "#@typeberry/logger";
+import { SerializedState, StateEntries } from "#@typeberry/state-merkleization";
+export const logger = Logger.new(import.meta.filename, "jam");
+export function getChainSpec(name) {
+    if (name === KnownChainSpec.Full) {
+        return fullChainSpec;
+    }
+    if (name === KnownChainSpec.Tiny) {
+        return tinyChainSpec;
+    }
+    throw new Error(`Unknown chain spec: ${name}. Possible options: ${[KnownChainSpec.Full, KnownChainSpec.Tiny]}`);
+}
+export function getDatabasePath(blake2b, nodeName, genesisHeader, databaseBasePath) {
+    const nodeNameHash = blake2b.hashString(nodeName).toString().substring(2, 10);
+    const genesisHeaderHash = blake2b.hashBytes(genesisHeader).asOpaque();
+    const genesisHeaderHashNibbles = genesisHeaderHash.toString().substring(2, 10);
+    const dbPath = `${databaseBasePath}/${nodeNameHash}/${genesisHeaderHashNibbles}`;
+    return {
+        dbPath,
+        genesisHeaderHash,
+    };
+}
+export async function initializeDatabase(spec, blake2b, genesisHeaderHash, rootDb, config, ancestry, options = {}) {
+    const blocks = rootDb.getBlocksDb();
+    const states = rootDb.getStatesDb();
+    const header = blocks.getBestHeaderHash();
+    const state = blocks.getPostStateRoot(header);
+    logger.log `🛢️ Best header hash: ${header}`;
+    logger.log `🛢️ Best state root: ${state}`;
+    // DB seems already initialized, just go with what we have.
+    const isDbInitialized = state !== null && !state.isEqualTo(Bytes.zero(HASH_SIZE)) && !header.isEqualTo(Bytes.zero(HASH_SIZE));
+    if (isDbInitialized) {
+        return;
+    }
+    logger.log `🛢️ Database looks fresh. Initializing.`;
+    // looks like a fresh db, initialize the state.
+    const genesisHeader = Decoder.decodeObject(Header.Codec, config.genesisHeader, spec);
+    const genesisExtrinsic = emptyBlock().extrinsic;
+    const genesisBlock = Block.create({ header: genesisHeader, extrinsic: genesisExtrinsic });
+    const blockView = reencodeAsView(Block.Codec, genesisBlock, spec);
+    logger.log `🧬 Writing genesis block #${genesisHeader.timeSlotIndex}: ${genesisHeaderHash}`;
+    const { genesisStateSerialized, genesisStateRootHash } = loadGenesisState(spec, blake2b, config.genesisState);
+    // write to db
+    // When initGenesisFromAncestry is set, use ancestry[0][0] as the initial block hash (for fuzz-target mode)
+    const initialBlockHash = (options.initGenesisFromAncestry ?? false) && ancestry.length > 0 ? ancestry[0][0] : genesisHeaderHash;
+    await blocks.insertBlock(new WithHash(initialBlockHash, blockView));
+    // insert fake blocks for ancestry data
+    for (const [hash, slot] of ancestry) {
+        await blocks.insertBlock(new WithHash(hash, reencodeAsView(Block.Codec, emptyBlock(slot), spec)));
+    }
+    await states.insertInitialState(initialBlockHash, genesisStateSerialized);
+    await blocks.setPostStateRoot(initialBlockHash, genesisStateRootHash);
+    await blocks.setBestHeaderHash(initialBlockHash);
+}
+function loadGenesisState(spec, blake2b, data) {
+    const stateEntries = StateEntries.fromEntriesUnsafe(data.entries());
+    const state = SerializedState.fromStateEntries(spec, blake2b, stateEntries);
+    const genesisStateRootHash = stateEntries.getRootHash(blake2b);
+    logger.info `🧬 Genesis state root: ${genesisStateRootHash}`;
+    return {
+        genesisState: state,
+        genesisStateSerialized: stateEntries,
+        genesisStateRootHash,
+    };
+}

package/packages/jam/node/export.d.ts

@@ -0,0 +1,3 @@
+import type { JamConfig } from "./jam-config.js";
+export declare function exportBlocks(jamNodeConfig: JamConfig, output: string, withRelPath: (p: string) => string): Promise<void>;
+//# sourceMappingURL=export.d.ts.map

package/packages/jam/node/export.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBA2F9G"}